From 76761fa33bd112d1fccf197941a23768d7d20c58 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 20 Jul 2021 16:26:29 +0300 Subject: linux-gen: dpdk: fix invalid mbuf data offset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix the check in mbuf_update() which detects if mbuf data offset needs to be updated. The previous check didn't detect a scenario where data offset was not at default value (caused by e.g. odp_packet_push_head()). Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen Reported-and-tested-by: Christian Hong --- platform/linux-generic/pktio/dpdk.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'platform') diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index a98324ee4..9abba6292 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -288,7 +288,7 @@ static inline void mbuf_update(struct rte_mbuf *mbuf, odp_packet_hdr_t *pkt_hdr, mbuf->refcnt = 1; mbuf->ol_flags = 0; - if (odp_unlikely(pkt_hdr->buf_hdr.base_data != pkt_hdr->seg_data)) + if (odp_unlikely(((uint8_t *)mbuf->buf_addr + mbuf->data_off) != pkt_hdr->seg_data)) mbuf->data_off = mbuf_data_off(mbuf, pkt_hdr); } -- cgit v1.2.3 From 7025d9960b4fd988de9b6631dce0829ba4eed238 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 29 Jul 2021 15:56:01 +0300 Subject: linux-gen: pktio: fix lto build error Fix LTO build error (maybe-uninitialized) detected by GCC 9.3.0. The linker doesn't understand capability checking correctly: pktio.c:2613:2: error: 'data.tx_pkt_drops' may be used uninitialized in this function [-Werror=maybe-uninitialized] 2613 | CU_ASSERT(!(capa->tx.counters.bit.tx_pkt_drops && (data.tx_pkt_drops > 0))); Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- platform/linux-generic/odp_packet_io.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 8dfd473ac..bd8bb58e8 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -3398,7 +3398,8 @@ int odp_proto_stats(odp_proto_stats_t stat, odp_proto_stats_data_t *data) { (void)stat; - (void)data; + + memset(data, 0, sizeof(odp_proto_stats_data_t)); return 0; } -- cgit v1.2.3 From cb8d8942bc54950b1981e0d99022b50e31771beb Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Fri, 20 Aug 2021 15:20:22 +0300 Subject: linux-gen: sched: limit interface header visibility Scheduler interface header should be included only when needed. Buffer internal header is included by many files, which do not use scheduler interface. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- platform/linux-generic/include/odp_buffer_internal.h | 1 - platform/linux-generic/odp_ipsec.c | 1 + platform/linux-generic/odp_traffic_mngr.c | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) (limited to 'platform') diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index d2e3c808f..e7bc78d6e 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -30,7 +30,6 @@ extern "C" { #include #include #include -#include #include typedef union buffer_index_t { diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 137e7b435..22ecfb35a 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index de01af96c..819204a41 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -32,6 +32,7 @@ #include #include #include +#include /* Local vars */ static const -- cgit v1.2.3 From c6f4b3a4605dbae21ed1c1973f07f408aeea54f1 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Fri, 13 Aug 2021 11:23:14 +0300 Subject: linux-gen: sched: add scheduler identifier Added scheduler identifier to the internal scheduler interface. This can be used e.g. in queue (basic) implementation to select actions based on the scheduler (basic vs sp). Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- platform/linux-generic/include/odp_schedule_if.h | 9 +++++++-- platform/linux-generic/odp_schedule_if.c | 5 +++++ 2 files changed, 12 insertions(+), 2 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index d3202543d..a804f8c95 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -16,8 +17,12 @@ extern "C" { #include #include -/* Number of ordered locks per queue */ -#define SCHEDULE_ORDERED_LOCKS_PER_QUEUE 2 +#define _ODP_SCHED_ID_BASIC 0 +#define _ODP_SCHED_ID_SP 1 +#define _ODP_SCHED_ID_SCALABLE 2 + +/* Scheduler identifier */ +extern int _odp_sched_id; typedef struct schedule_config_t { struct { diff --git a/platform/linux-generic/odp_schedule_if.c b/platform/linux-generic/odp_schedule_if.c index 24307ab90..528c4065e 100644 --- a/platform/linux-generic/odp_schedule_if.c +++ b/platform/linux-generic/odp_schedule_if.c @@ -1,4 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -25,6 +26,7 @@ extern const schedule_api_t _odp_schedule_scalable_api; const schedule_fn_t *_odp_sched_fn; const schedule_api_t *_odp_sched_api; +int _odp_sched_id; uint64_t odp_schedule_wait_time(uint64_t ns) { @@ -216,12 +218,15 @@ int _odp_schedule_init_global(void) ODP_PRINT("Using scheduler '%s'\n", sched); if (!strcmp(sched, "basic")) { + _odp_sched_id = _ODP_SCHED_ID_BASIC; _odp_sched_fn = &_odp_schedule_basic_fn; _odp_sched_api = &_odp_schedule_basic_api; } else if (!strcmp(sched, "sp")) { + _odp_sched_id = _ODP_SCHED_ID_SP; _odp_sched_fn = &_odp_schedule_sp_fn; _odp_sched_api = &_odp_schedule_sp_api; } else if (!strcmp(sched, "scalable")) { + _odp_sched_id = _ODP_SCHED_ID_SCALABLE; _odp_sched_fn = &_odp_schedule_scalable_fn; _odp_sched_api = &_odp_schedule_scalable_api; } else { -- cgit v1.2.3 From 3c40dfc7a8ea0cc0e6679613ad9e22163c2771e7 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Fri, 13 Aug 2021 12:34:42 +0300 Subject: linux-gen: queue: improve queue debug print Added scheduler group and spread to debug print output. Spread is printed only with basic scheduler (SP does not have it). Changed queue debug print output to match odp_schedule_print() output, which prints scheduler internal priority level (0 == highest prio). Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- .../include/odp_queue_basic_internal.h | 3 +++ platform/linux-generic/odp_queue_basic.c | 31 +++++++++++++++++----- platform/linux-generic/odp_schedule_basic.c | 6 +++++ 3 files changed, 34 insertions(+), 6 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/include/odp_queue_basic_internal.h b/platform/linux-generic/include/odp_queue_basic_internal.h index 25e35b22c..60817fc75 100644 --- a/platform/linux-generic/include/odp_queue_basic_internal.h +++ b/platform/linux-generic/include/odp_queue_basic_internal.h @@ -119,6 +119,9 @@ int _odp_sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int num, int update_status); int _odp_sched_queue_empty(uint32_t queue_index); +/* Functions by schedulers */ +int _odp_sched_basic_get_spread(uint32_t queue_index); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 428022ec8..76f897dfd 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -677,7 +677,8 @@ static void queue_print(odp_queue_t handle) odp_pktio_info_t pktio_info; queue_entry_t *queue; uint32_t queue_id; - int status; + int status, prio; + int max_prio = odp_schedule_max_prio(); queue_id = queue_to_index(handle); @@ -701,7 +702,7 @@ static void queue_print(odp_queue_t handle) ODP_PRINT("\nQueue info\n"); ODP_PRINT("----------\n"); ODP_PRINT(" handle %p\n", queue->s.handle); - ODP_PRINT(" index %" PRIu32 "\n", queue->s.index); + ODP_PRINT(" index %" PRIu32 "\n", queue_id); ODP_PRINT(" name %s\n", queue->s.name); ODP_PRINT(" enq mode %s\n", queue->s.param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : @@ -729,8 +730,11 @@ static void queue_print(odp_queue_t handle) "ODP_SCHED_SYNC_ATOMIC" : (queue->s.param.sched.sync == ODP_SCHED_SYNC_ORDERED ? "ODP_SCHED_SYNC_ORDERED" : "unknown"))); - ODP_PRINT(" priority %d\n", queue->s.param.sched.prio); - ODP_PRINT(" group %d\n", queue->s.param.sched.group); + prio = queue->s.param.sched.prio; + ODP_PRINT(" priority %i (%i in API)\n", max_prio - prio, prio); + ODP_PRINT(" group %i\n", queue->s.param.sched.group); + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + ODP_PRINT(" spread %i\n", _odp_sched_basic_get_spread(queue_id)); } if (queue->s.pktin.pktio != ODP_PKTIO_INVALID) { if (!odp_pktio_info(queue->s.pktin.pktio, &pktio_info)) @@ -784,11 +788,17 @@ static void queue_print_all(void) char type_c, enq_c, deq_c, order_c, sync_c; const int col_width = 24; int prio = 0; + int spr = 0; odp_schedule_sync_t sync = ODP_SCHED_SYNC_PARALLEL; + odp_schedule_group_t grp = ODP_SCHED_GROUP_INVALID; ODP_PRINT("\nList of all queues\n"); ODP_PRINT("------------------\n"); - ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio\n", col_width, "name"); + ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio grp", col_width, "name"); + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + ODP_PRINT(" spr\n"); + else + ODP_PRINT("\n"); for (i = 0; i < CONFIG_MAX_QUEUES; i++) { queue_entry_t *queue = qentry_from_index(i); @@ -817,7 +827,10 @@ static void queue_print_all(void) len = ring_st_length(&queue->s.ring_st); max_len = queue->s.ring_mask + 1; prio = queue->s.param.sched.prio; + grp = queue->s.param.sched.group; sync = queue->s.param.sched.sync; + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + spr = _odp_sched_basic_get_spread(index); } else { len = ring_mpmc_length(&queue->s.ring_mpmc); max_len = queue->s.ring_mask + 1; @@ -851,7 +864,13 @@ static void queue_print_all(void) if (type == ODP_QUEUE_TYPE_SCHED) { sync_c = (sync == ODP_SCHED_SYNC_PARALLEL) ? 'P' : ((sync == ODP_SCHED_SYNC_ATOMIC) ? 'A' : 'O'); - ODP_PRINT(" %c %4i", sync_c, prio); + /* Print prio level matching odp_schedule_print() output */ + prio = odp_schedule_max_prio() - prio; + + ODP_PRINT(" %c %4i %3i", sync_c, prio, grp); + + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + ODP_PRINT(" %3i", spr); } ODP_PRINT("\n"); diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 479d6f956..df8dade8f 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -1947,6 +1947,12 @@ static void schedule_print(void) ODP_PRINT("\n"); } +/* Returns spread for queue debug prints */ +int _odp_sched_basic_get_spread(uint32_t queue_index) +{ + return sched->queue[queue_index].spread; +} + /* Fill in scheduler interface */ const schedule_fn_t _odp_schedule_basic_fn = { .pktio_start = schedule_pktio_start, -- cgit v1.2.3 From e76de56d415396074d6eddbf2236a0b709a03519 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 19 Aug 2021 08:16:17 +0300 Subject: linux-gen: sched: update thread count on group create Update number of threads per spread statistics on group create and destroy. A thread can be placed to a group with create or join calls. Dynamic scheduler uses thread counts and did not work correctly, if threads were added to a group with the create call. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- platform/linux-generic/odp_schedule_basic.c | 104 +++++++++++++++++++++------- 1 file changed, 78 insertions(+), 26 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index df8dade8f..3d387cec9 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -1573,11 +1573,64 @@ static uint64_t schedule_wait_time(uint64_t ns) return ns; } +static inline void spread_thrs_inc(odp_schedule_group_t group, int thr_tbl[], int count) +{ + int thr, i; + uint8_t spread; + + for (i = 0; i < count; i++) { + thr = thr_tbl[i]; + spread = spread_from_index(thr); + sched->sched_grp[group].spread_thrs[spread]++; + } +} + +static inline void spread_thrs_dec(odp_schedule_group_t group, int thr_tbl[], int count) +{ + int thr, i; + uint8_t spread; + + for (i = 0; i < count; i++) { + thr = thr_tbl[i]; + spread = spread_from_index(thr); + sched->sched_grp[group].spread_thrs[spread]--; + } +} + +static inline int threads_from_mask(int thr_tbl[], int count, const odp_thrmask_t *mask) +{ + int i; + int thr = odp_thrmask_first(mask); + + for (i = 0; i < count; i++) { + if (thr < 0) { + ODP_ERR("No more threads in the mask\n"); + return -1; + } + + thr_tbl[i] = thr; + thr = odp_thrmask_next(mask, thr); + } + + return 0; +} + static odp_schedule_group_t schedule_group_create(const char *name, const odp_thrmask_t *mask) { odp_schedule_group_t group = ODP_SCHED_GROUP_INVALID; - int i; + int count, i; + + count = odp_thrmask_count(mask); + if (count < 0) { + ODP_ERR("Bad thread count\n"); + return ODP_SCHED_GROUP_INVALID; + } + + int thr_tbl[count]; + + if (count && threads_from_mask(thr_tbl, count, mask)) + return ODP_SCHED_GROUP_INVALID; odp_ticketlock_lock(&sched->grp_lock); @@ -1595,6 +1648,7 @@ static odp_schedule_group_t schedule_group_create(const char *name, grp_update_mask(i, mask); group = (odp_schedule_group_t)i; + spread_thrs_inc(group, thr_tbl, count); sched->sched_grp[i].allocated = 1; break; } @@ -1607,25 +1661,33 @@ static odp_schedule_group_t schedule_group_create(const char *name, static int schedule_group_destroy(odp_schedule_group_t group) { odp_thrmask_t zero; - int ret; + int i; + + if (group >= NUM_SCHED_GRPS || group < SCHED_GROUP_NAMED) { + ODP_ERR("Bad group %i\n", group); + return -1; + } odp_thrmask_zero(&zero); odp_ticketlock_lock(&sched->grp_lock); - if (group < NUM_SCHED_GRPS && group >= SCHED_GROUP_NAMED && - sched->sched_grp[group].allocated) { - grp_update_mask(group, &zero); - memset(sched->sched_grp[group].name, 0, - ODP_SCHED_GROUP_NAME_LEN); - sched->sched_grp[group].allocated = 0; - ret = 0; - } else { - ret = -1; + if (sched->sched_grp[group].allocated == 0) { + odp_ticketlock_unlock(&sched->grp_lock); + ODP_ERR("Group not created: %i\n", group); + return -1; } + grp_update_mask(group, &zero); + + for (i = 0; i < MAX_SPREAD; i++) + sched->sched_grp[group].spread_thrs[i] = 0; + + memset(sched->sched_grp[group].name, 0, ODP_SCHED_GROUP_NAME_LEN); + sched->sched_grp[group].allocated = 0; + odp_ticketlock_unlock(&sched->grp_lock); - return ret; + return 0; } static odp_schedule_group_t schedule_group_lookup(const char *name) @@ -1649,7 +1711,6 @@ static odp_schedule_group_t schedule_group_lookup(const char *name) static int schedule_group_join(odp_schedule_group_t group, const odp_thrmask_t *mask) { int i, count, thr; - uint8_t spread; odp_thrmask_t new_mask; if (group >= NUM_SCHED_GRPS || group < SCHED_GROUP_NAMED) { @@ -1684,10 +1745,7 @@ static int schedule_group_join(odp_schedule_group_t group, const odp_thrmask_t * return -1; } - for (i = 0; i < count; i++) { - spread = spread_from_index(thr_tbl[i]); - sched->sched_grp[group].spread_thrs[spread]++; - } + spread_thrs_inc(group, thr_tbl, count); odp_thrmask_or(&new_mask, &sched->sched_grp[group].mask, mask); grp_update_mask(group, &new_mask); @@ -1699,7 +1757,6 @@ static int schedule_group_join(odp_schedule_group_t group, const odp_thrmask_t * static int schedule_group_leave(odp_schedule_group_t group, const odp_thrmask_t *mask) { int i, count, thr; - uint8_t spread; odp_thrmask_t new_mask; if (group >= NUM_SCHED_GRPS || group < SCHED_GROUP_NAMED) { @@ -1736,10 +1793,7 @@ static int schedule_group_leave(odp_schedule_group_t group, const odp_thrmask_t return -1; } - for (i = 0; i < count; i++) { - spread = spread_from_index(thr_tbl[i]); - sched->sched_grp[group].spread_thrs[spread]--; - } + spread_thrs_dec(group, thr_tbl, count); odp_thrmask_and(&new_mask, &sched->sched_grp[group].mask, &new_mask); grp_update_mask(group, &new_mask); @@ -1789,7 +1843,6 @@ static int schedule_thr_add(odp_schedule_group_t group, int thr) { odp_thrmask_t mask; odp_thrmask_t new_mask; - uint8_t spread = spread_from_index(thr); if (group < 0 || group >= SCHED_GROUP_NAMED) return -1; @@ -1805,7 +1858,7 @@ static int schedule_thr_add(odp_schedule_group_t group, int thr) } odp_thrmask_or(&new_mask, &sched->sched_grp[group].mask, &mask); - sched->sched_grp[group].spread_thrs[spread]++; + spread_thrs_inc(group, &thr, 1); grp_update_mask(group, &new_mask); odp_ticketlock_unlock(&sched->grp_lock); @@ -1817,7 +1870,6 @@ static int schedule_thr_rem(odp_schedule_group_t group, int thr) { odp_thrmask_t mask; odp_thrmask_t new_mask; - uint8_t spread = spread_from_index(thr); if (group < 0 || group >= SCHED_GROUP_NAMED) return -1; @@ -1834,7 +1886,7 @@ static int schedule_thr_rem(odp_schedule_group_t group, int thr) } odp_thrmask_and(&new_mask, &sched->sched_grp[group].mask, &new_mask); - sched->sched_grp[group].spread_thrs[spread]--; + spread_thrs_dec(group, &thr, 1); grp_update_mask(group, &new_mask); odp_ticketlock_unlock(&sched->grp_lock); -- cgit v1.2.3 From bd28195606cf9c2101c890ae2adca677af598766 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Fri, 20 Aug 2021 11:05:33 +0300 Subject: linux-gen: sched: add randomness to queue create spread selection When there are multiple spreads with the minimum number of queues, select the spread randomly among those. This makes spreads more evenly loaded, especially when each group/prio has only few queues (e.g. many groups/prios, but only 1-2 queues per group/prio). Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- platform/linux-generic/odp_schedule_basic.c | 61 +++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 4 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 3d387cec9..ef0a880b6 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -78,6 +78,9 @@ ODP_STATIC_ASSERT((QUEUE_LOAD * CONFIG_MAX_SCHED_QUEUES) < UINT32_MAX, "Load_val /* Spread weight table */ #define SPREAD_TBL_SIZE ((MAX_SPREAD - 1) * MAX_PREFER_RATIO) +/* Random data table size */ +#define RANDOM_TBL_SIZE 128 + /* Maximum number of packet IO interfaces */ #define NUM_PKTIO ODP_CONFIG_PKTIO_ENTRIES @@ -133,6 +136,27 @@ typedef union { ODP_STATIC_ASSERT(sizeof(lock_called_t) == sizeof(uint32_t), "Lock_called_values_do_not_fit_in_uint32"); +static uint8_t sched_random_u8[] = { + 0x64, 0xe3, 0x64, 0x0a, 0x0a, 0x5b, 0x7e, 0xd7, + 0x43, 0xb7, 0x90, 0x71, 0x76, 0x17, 0x8e, 0x3f, + 0x17, 0x60, 0x7e, 0xfd, 0x99, 0xe3, 0xab, 0x06, + 0x77, 0xf9, 0x45, 0x17, 0x2f, 0x81, 0x9e, 0x7b, + 0x20, 0x1b, 0x36, 0x75, 0x69, 0xc5, 0x69, 0x27, + 0x7a, 0xf6, 0x3f, 0x63, 0x2c, 0x3f, 0x1b, 0xeb, + 0x12, 0xe1, 0x6f, 0xd4, 0xd9, 0x14, 0x97, 0xa6, + 0x2a, 0xe5, 0xb0, 0x45, 0x27, 0xa6, 0x48, 0xbc, + 0x2b, 0xec, 0xd8, 0xda, 0x55, 0xef, 0x15, 0xce, + 0xf8, 0xc2, 0x1e, 0xc8, 0x16, 0x6c, 0xf0, 0x4f, + 0x1a, 0xc7, 0x50, 0x9e, 0x0b, 0xa5, 0xe9, 0xf3, + 0x28, 0x79, 0x2e, 0x18, 0xb0, 0xb4, 0xac, 0xce, + 0x67, 0x04, 0x52, 0x98, 0xce, 0x8c, 0x05, 0x87, + 0xab, 0xc8, 0x94, 0x7e, 0x46, 0x63, 0x60, 0x8d, + 0x3d, 0x8f, 0x14, 0x85, 0x1e, 0x92, 0xd2, 0x40, + 0x2d, 0x42, 0xfe, 0xf1, 0xc2, 0xb6, 0x03, 0x43 +}; + +ODP_STATIC_ASSERT(sizeof(sched_random_u8) == RANDOM_TBL_SIZE, "Bad_random_table_size"); + /* Scheduler local data */ typedef struct ODP_ALIGNED_CACHE { uint32_t sched_round; @@ -247,6 +271,7 @@ typedef struct { /* Scheduler interface config options (not used in fast path) */ schedule_config_t config_if; uint32_t max_queues; + odp_atomic_u32_t next_rand; } sched_global_t; @@ -500,6 +525,7 @@ static int schedule_init_global(void) odp_ticketlock_init(&sched->grp_lock); odp_atomic_init_u32(&sched->grp_epoch, 0); + odp_atomic_init_u32(&sched->next_rand, 0); for (i = 0; i < NUM_SCHED_GRPS; i++) { memset(sched->sched_grp[i].name, 0, ODP_SCHED_GROUP_NAME_LEN); @@ -662,25 +688,42 @@ static inline void update_queue_count(int grp, int prio, int old_spr, int new_sp odp_ticketlock_unlock(&sched->mask_lock[grp]); } -/* Select the spread that has least queues */ static uint8_t allocate_spread(int grp, int prio) { - uint8_t i; + uint8_t i, num_min, spr; uint32_t num; uint32_t min = UINT32_MAX; uint8_t num_spread = sched->config.num_spread; - uint8_t spr = 0; + uint8_t min_spr[num_spread]; + + num_min = 1; + min_spr[0] = 0; odp_ticketlock_lock(&sched->mask_lock[grp]); + /* Find spread(s) with the minimum number of queues */ for (i = 0; i < num_spread; i++) { num = sched->prio_q_count[grp][prio][i]; if (num < min) { - spr = i; min = num; + min_spr[0] = i; + num_min = 1; + } else if (num == min) { + min_spr[num_min] = i; + num_min++; } } + spr = min_spr[0]; + + /* When there are multiple minimum spreads, select one randomly */ + if (num_min > 1) { + uint32_t next_rand = odp_atomic_fetch_inc_u32(&sched->next_rand); + uint8_t rand = sched_random_u8[next_rand % RANDOM_TBL_SIZE]; + + spr = min_spr[rand % num_min]; + } + sched->prio_q_mask[grp][prio] |= 1 << spr; sched->prio_q_count[grp][prio][spr]++; @@ -719,6 +762,16 @@ static int schedule_create_queue(uint32_t queue_index, return -1; } + odp_ticketlock_lock(&sched->grp_lock); + + if (sched->sched_grp[grp].allocated == 0) { + odp_ticketlock_unlock(&sched->grp_lock); + ODP_ERR("Group not created: %i\n", grp); + return -1; + } + + odp_ticketlock_unlock(&sched->grp_lock); + spread = allocate_spread(grp, prio); sched->queue[queue_index].grp = grp; -- cgit v1.2.3 From 83c800e69852219e0dd16448dd00bf13aec04e68 Mon Sep 17 00:00:00 2001 From: Vijay Ram Inavolu Date: Wed, 18 Aug 2021 15:12:18 +0000 Subject: linux-gen: ipsec: implement odp_ipsec_stats_multi() Implement odp_ipsec_stats_multi() which is currently missing. Signed-off-by: Vijay Ram Inavolu Reviewed-by: Janne Peltonen --- platform/linux-generic/odp_ipsec.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'platform') diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 22ecfb35a..9b05cc8c3 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -2197,6 +2197,21 @@ int odp_ipsec_stats(odp_ipsec_sa_t sa, odp_ipsec_stats_t *stats) return 0; } +int odp_ipsec_stats_multi(odp_ipsec_sa_t sa[], odp_ipsec_stats_t stats[], int num) +{ + int ret, i; + + ODP_ASSERT(NULL != stats); + + for (i = 0; i < num; i++) { + ret = odp_ipsec_stats(sa[i], &stats[i]); + if (ret) + return ret; + } + + return 0; +} + static int read_config_file(ipsec_global_t *global) { const char *str_i = "ipsec.ordering.async_inbound"; -- cgit v1.2.3 From 589a87f9ee17bde8af9d472715cd3ede09ac30a2 Mon Sep 17 00:00:00 2001 From: Vijay Ram Inavolu Date: Wed, 18 Aug 2021 14:05:57 +0000 Subject: linux-gen: ipsec: fix for flow & tos values in ipv6hdr These values are not updated for ipv6 tunnel correctly due to endianness issue Signed-off-by: Vijay Ram Inavolu Reviewed-by: Janne Peltonen --- platform/linux-generic/odp_ipsec.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 9b05cc8c3..572c67747 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -1015,13 +1015,14 @@ static int ipsec_out_tunnel_parse_ipv6(ipsec_state_t *state, ipsec_sa_t *ipsec_sa) { _odp_ipv6hdr_t *ipv6hdr = state->ip; + uint32_t ver_tc_flow = odp_be_to_cpu_32(ipv6hdr->ver_tc_flow); ipv6hdr->hop_limit -= ipsec_sa->dec_ttl; - state->out_tunnel.ip_tos = (ipv6hdr->ver_tc_flow & + state->out_tunnel.ip_tos = (ver_tc_flow & _ODP_IPV6HDR_TC_MASK) >> _ODP_IPV6HDR_TC_SHIFT; state->out_tunnel.ip_df = 0; - state->out_tunnel.ip_flabel = (ipv6hdr->ver_tc_flow & + state->out_tunnel.ip_flabel = (ver_tc_flow & _ODP_IPV6HDR_FLOW_LABEL_MASK) >> _ODP_IPV6HDR_FLOW_LABEL_SHIFT; state->ip_next_hdr = ipv6hdr->next_hdr; -- cgit v1.2.3 From 35c21145c5b13b185b8010d1ebf904f0bb51ee1b Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 27 Aug 2021 09:26:38 +0300 Subject: linux-gen: crypto: fix error handling in odp_crypto_operation() odp_crypto_operation() calls odp_crypto_op() but incorrectly interprets zero return value as success when it actually means that zero packets were successfully processed. Consequently the result structure gets filled with data that was not set by the failed odp_crypto_op(), input packet may leak and output packet may not exists or contain incorrect data. Fix the problem by returning an error if odp_crypto_op() returns zero. Also, do not return a failure when odp_crypto_result() fails. At that point odp_crypto_op() has already done the processing, including freeing or overwriting the input packet. Indicate failure through the result structure instead. Currently odp_crypto_result() never fails, but it may not stay that way (and this piece of code seems to get reused elsewhere). Signed-off-by: Janne Peltonen Reviewed-by: Matias Elo --- platform/linux-generic/odp_crypto_null.c | 14 ++++++++++---- platform/linux-generic/odp_crypto_openssl.c | 14 ++++++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index cf9abc99f..eb8085d40 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -276,12 +276,18 @@ odp_crypto_operation(odp_crypto_op_param_t *param, packet_param.auth_range = param->auth_range; rc = odp_crypto_op(¶m->pkt, &out_pkt, &packet_param, 1); - if (rc < 0) - return rc; + if (rc <= 0) + return -1; rc = odp_crypto_result(&packet_result, out_pkt); - if (rc < 0) - return rc; + if (rc < 0) { + /* + * We cannot fail since odp_crypto_op() has already processed + * the packet. Let's indicate error in the result instead. + */ + packet_hdr(out_pkt)->p.flags.crypto_err = 1; + packet_result.ok = false; + } /* Indicate to caller operation was sync */ *posted = 0; diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 07a91cc46..6cad38f92 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -2461,12 +2461,18 @@ odp_crypto_operation(odp_crypto_op_param_t *param, packet_param.auth_range = param->auth_range; rc = odp_crypto_op(¶m->pkt, &out_pkt, &packet_param, 1); - if (rc < 0) - return rc; + if (rc <= 0) + return -1; rc = odp_crypto_result(&packet_result, out_pkt); - if (rc < 0) - return rc; + if (rc < 0) { + /* + * We cannot fail since odp_crypto_op() has already processed + * the packet. Let's indicate error in the result instead. + */ + packet_hdr(out_pkt)->p.flags.crypto_err = 1; + packet_result.ok = false; + } /* Indicate to caller operation was sync */ *posted = 0; -- cgit v1.2.3 From 0fba404bb988058ff1d7a7655cce82dfa95a3e84 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 25 Aug 2021 16:58:41 +0300 Subject: linux-gen: atomic: remove duplicate 64bit inline operation defines 64bit atomic inline operation defines were added twice. Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- .../include/odp/api/plat/atomic_inlines.h | 34 +++++----------------- 1 file changed, 7 insertions(+), 27 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h index 4ab8bb411..a9da70890 100644 --- a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h @@ -34,6 +34,13 @@ #define odp_atomic_dec_u32 __odp_atomic_dec_u32 #define odp_atomic_cas_u32 __odp_atomic_cas_u32 #define odp_atomic_xchg_u32 __odp_atomic_xchg_u32 + #define odp_atomic_load_acq_u32 __odp_atomic_load_acq_u32 + #define odp_atomic_store_rel_u32 __odp_atomic_store_rel_u32 + #define odp_atomic_add_rel_u32 __odp_atomic_add_rel_u32 + #define odp_atomic_sub_rel_u32 __odp_atomic_sub_rel_u32 + #define odp_atomic_cas_acq_u32 __odp_atomic_cas_acq_u32 + #define odp_atomic_cas_rel_u32 __odp_atomic_cas_rel_u32 + #define odp_atomic_cas_acq_rel_u32 __odp_atomic_cas_acq_rel_u32 #define odp_atomic_max_u32 __odp_atomic_max_u32 #define odp_atomic_min_u32 __odp_atomic_min_u32 #define odp_atomic_init_u64 __odp_atomic_init_u64 @@ -56,35 +63,8 @@ #define odp_atomic_cas_acq_u64 __odp_atomic_cas_acq_u64 #define odp_atomic_cas_rel_u64 __odp_atomic_cas_rel_u64 #define odp_atomic_cas_acq_rel_u64 __odp_atomic_cas_acq_rel_u64 - #define odp_atomic_init_u64 __odp_atomic_init_u64 - #define odp_atomic_load_u64 __odp_atomic_load_u64 - #define odp_atomic_store_u64 __odp_atomic_store_u64 - #define odp_atomic_fetch_add_u64 __odp_atomic_fetch_add_u64 - #define odp_atomic_add_u64 __odp_atomic_add_u64 - #define odp_atomic_fetch_sub_u64 __odp_atomic_fetch_sub_u64 - #define odp_atomic_sub_u64 __odp_atomic_sub_u64 - #define odp_atomic_fetch_inc_u64 __odp_atomic_fetch_inc_u64 - #define odp_atomic_inc_u64 __odp_atomic_inc_u64 - #define odp_atomic_fetch_dec_u64 __odp_atomic_fetch_dec_u64 - #define odp_atomic_dec_u64 __odp_atomic_dec_u64 - #define odp_atomic_cas_u64 __odp_atomic_cas_u64 - #define odp_atomic_xchg_u64 __odp_atomic_xchg_u64 - #define odp_atomic_load_acq_u64 __odp_atomic_load_acq_u64 - #define odp_atomic_store_rel_u64 __odp_atomic_store_rel_u64 - #define odp_atomic_add_rel_u64 __odp_atomic_add_rel_u64 - #define odp_atomic_sub_rel_u64 __odp_atomic_sub_rel_u64 - #define odp_atomic_cas_acq_u64 __odp_atomic_cas_acq_u64 - #define odp_atomic_cas_rel_u64 __odp_atomic_cas_rel_u64 - #define odp_atomic_cas_acq_rel_u64 __odp_atomic_cas_acq_rel_u64 #define odp_atomic_max_u64 __odp_atomic_max_u64 #define odp_atomic_min_u64 __odp_atomic_min_u64 - #define odp_atomic_load_acq_u32 __odp_atomic_load_acq_u32 - #define odp_atomic_store_rel_u32 __odp_atomic_store_rel_u32 - #define odp_atomic_add_rel_u32 __odp_atomic_add_rel_u32 - #define odp_atomic_sub_rel_u32 __odp_atomic_sub_rel_u32 - #define odp_atomic_cas_acq_u32 __odp_atomic_cas_acq_u32 - #define odp_atomic_cas_rel_u32 __odp_atomic_cas_rel_u32 - #define odp_atomic_cas_acq_rel_u32 __odp_atomic_cas_acq_rel_u32 #define odp_atomic_init_u128 __odp_atomic_init_u128 #define odp_atomic_load_u128 __odp_atomic_load_u128 #define odp_atomic_store_u128 __odp_atomic_store_u128 -- cgit v1.2.3 From 7076b6011ebe4b47377a6eb24775d083ad2e20d0 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 25 Aug 2021 17:13:44 +0300 Subject: linux-gen: atomic: add missing inline function include Previously, inlined atomic functions (atomic_inlines.h) were not included through the API headers in non-ABI compat mode on aarch64 platforms. Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'platform') diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h index d1dbf36b8..14cca3ca0 100644 --- a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h @@ -9,4 +9,4 @@ #endif #include - +#include -- cgit v1.2.3 From 28266184f52a7b12bb32256270896e08ca87eaba Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 26 Aug 2021 08:54:11 +0300 Subject: linux-gen: remove duplicate _ODP_INLINE defines _ODP_INLINE defines belong inside the *_inlines.h headers. Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- platform/linux-generic/include-abi/odp/api/abi/atomic.h | 1 - platform/linux-generic/include-abi/odp/api/abi/std.h | 1 - platform/linux-generic/include-abi/odp/api/abi/sync.h | 1 - 3 files changed, 3 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/include-abi/odp/api/abi/atomic.h b/platform/linux-generic/include-abi/odp/api/abi/atomic.h index 7c11b0ab2..9c87f9cb8 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/atomic.h +++ b/platform/linux-generic/include-abi/odp/api/abi/atomic.h @@ -84,7 +84,6 @@ typedef struct ODP_ALIGNED(sizeof(odp_u128_t)) odp_atomic_u128_s { * @{ */ -#define _ODP_INLINE static inline #include /** diff --git a/platform/linux-generic/include-abi/odp/api/abi/std.h b/platform/linux-generic/include-abi/odp/api/abi/std.h index 175b606c5..201fca18e 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/std.h +++ b/platform/linux-generic/include-abi/odp/api/abi/std.h @@ -17,7 +17,6 @@ extern "C" { #endif -#define _ODP_INLINE static inline #include #ifdef __cplusplus diff --git a/platform/linux-generic/include-abi/odp/api/abi/sync.h b/platform/linux-generic/include-abi/odp/api/abi/sync.h index cbb6f753e..236e92c8c 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/sync.h +++ b/platform/linux-generic/include-abi/odp/api/abi/sync.h @@ -21,7 +21,6 @@ extern "C" { * @{ */ -#define _ODP_INLINE static inline #include /** -- cgit v1.2.3 From be52a060a0d5ce60e47465a67f57106f0628e47d Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Mon, 13 Sep 2021 11:25:28 +0300 Subject: linux-gen: pool: fix cache alignment definition Fix cache alignment definition in pool_t. The correct definition to use is ODP_ALIGNED_CACHE (not ODP_CACHE_ALIGNED). Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- platform/linux-generic/include/odp_pool_internal.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'platform') diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index dc4754710..dc9f0d207 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -20,6 +20,7 @@ extern "C" { #include #include +#include #include #include @@ -94,7 +95,7 @@ typedef struct pool_t { pool_destroy_cb_fn ext_destroy; void *ext_desc; - struct ODP_CACHE_ALIGNED { + struct ODP_ALIGNED_CACHE { odp_atomic_u64_t alloc_ops; odp_atomic_u64_t alloc_fails; odp_atomic_u64_t free_ops; -- cgit v1.2.3 From 66c8e5f54bdb7c67628db8565c57803b8c200ce8 Mon Sep 17 00:00:00 2001 From: Vijay Ram Inavolu Date: Thu, 26 Aug 2021 11:05:05 +0000 Subject: linux-gen: config: add string array lookup function Adds a new function to get the size of the string array and its elements. Signed-off-by: Vijay Ram Inavolu Reviewed-by: Matias Elo --- .../linux-generic/include/odp_libconfig_internal.h | 2 + platform/linux-generic/odp_libconfig.c | 70 ++++++++++++++++++++++ 2 files changed, 72 insertions(+) (limited to 'platform') diff --git a/platform/linux-generic/include/odp_libconfig_internal.h b/platform/linux-generic/include/odp_libconfig_internal.h index ed80d1417..cfbdd541d 100644 --- a/platform/linux-generic/include/odp_libconfig_internal.h +++ b/platform/linux-generic/include/odp_libconfig_internal.h @@ -22,6 +22,8 @@ int _odp_libconfig_term_global(void); int _odp_libconfig_lookup_int(const char *path, int *value); int _odp_libconfig_lookup_array(const char *path, int value[], int max_num); +int _odp_libconfig_lookup_array_str(const char *path, char **value, + int max_count, unsigned int max_str); int _odp_libconfig_lookup_ext_int(const char *base_path, const char *local_path, diff --git a/platform/linux-generic/odp_libconfig.c b/platform/linux-generic/odp_libconfig.c index 93095ed1a..c23466624 100644 --- a/platform/linux-generic/odp_libconfig.c +++ b/platform/linux-generic/odp_libconfig.c @@ -1,5 +1,6 @@ /* Copyright (c) 2018, Linaro Limited * Copyright (c) 2020, Nokia + * Copyright (c) 2021, Marvell * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -111,6 +112,75 @@ int _odp_libconfig_lookup_int(const char *path, int *value) return (ret_def == CONFIG_TRUE || ret_rt == CONFIG_TRUE) ? 1 : 0; } +/** + * String array setting values + * + * Returns the number of strings in a string array setting. A valid runtime + * setting overrides default even if the array is empty. Outputs upto + * 'count' strings when the 'value' array pointer is not NULL. If return + * value is larger than 'count', there are more strings than the function was + * allowed to output. If return value (N) is less than 'count', only + * strings[0 ... N-1] have been written. + * + * @param path Path of the setting + * @param[out] value Array of strings to be copied from the setting + * @param count Number of strings to be copied + * @param str_size Maximum string length to be copied + * + * @return Number of strings in the setting + * @retval <0 on failure +*/ +int _odp_libconfig_lookup_array_str(const char *path, char **value, + int count, unsigned int str_size) +{ + config_setting_t *setting, *elem; + const config_t *config; + int num, i, j; + + for (i = 0; i < 2; i++) { + if (i == 0) + config = &odp_global_ro.libconfig_runtime; + else + config = &odp_global_ro.libconfig_default; + + setting = config_lookup(config, path); + + if (setting == NULL) + continue; + + /* invalid config if element is not an array */ + if (config_setting_is_array(setting) == CONFIG_FALSE) { + ODP_ERR("libconfig: %s is not an array\n", path); + return -1; + } + num = config_setting_length(setting); + + if (num == 0 || count == 0 || value == NULL) + return num; + + elem = config_setting_get_elem(setting, 0); + if (config_setting_type(elem) != CONFIG_TYPE_STRING) { + ODP_ERR("libconfig: %s array is not of type string\n", path); + return -1; + } + + for (j = 0; j < num; j++) { + elem = config_setting_get_elem(setting, j); + if (strlen(elem->value.sval) > str_size) { + ODP_ERR("libconfig: length of %s bigger than size %u\n", + elem->value.sval, str_size); + return -1; + } + strcpy(value[j], elem->value.sval); + } + + return num; + } + + ODP_ERR("libconfig: %s is not defined in config files\n", path); + return -1; +} + int _odp_libconfig_lookup_array(const char *path, int value[], int max_num) { const config_t *config; -- cgit v1.2.3 From f36c9ec6070071104cdc3d5b87568b28e9551040 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 20 Sep 2021 13:50:03 +0300 Subject: linux-gen: shm: rename implementation internal shm blocks Make implementation internal global data SHM block naming consistent (_odp_*_global). Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- platform/linux-generic/odp_comp.c | 2 +- platform/linux-generic/odp_crypto_null.c | 4 ++-- platform/linux-generic/odp_crypto_openssl.c | 4 ++-- platform/linux-generic/odp_ipsec.c | 4 ++-- platform/linux-generic/odp_pcapng.c | 2 +- platform/linux-generic/odp_pool.c | 2 +- platform/linux-generic/odp_queue_basic.c | 4 ++-- platform/linux-generic/odp_queue_lf.c | 2 +- platform/linux-generic/odp_schedule_basic.c | 2 +- platform/linux-generic/odp_schedule_scalable.c | 2 +- platform/linux-generic/odp_schedule_sp.c | 2 +- platform/linux-generic/odp_stash.c | 2 +- platform/linux-generic/odp_thread.c | 4 ++-- platform/linux-generic/odp_timer.c | 2 +- platform/linux-generic/odp_traffic_mngr.c | 2 +- 15 files changed, 20 insertions(+), 20 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_comp.c b/platform/linux-generic/odp_comp.c index 685c9098a..579bdc03b 100644 --- a/platform/linux-generic/odp_comp.c +++ b/platform/linux-generic/odp_comp.c @@ -620,7 +620,7 @@ int _odp_comp_init_global(void) mem_size = sizeof(*global); /* Allocate our globally shared memory */ - shm = odp_shm_reserve("_odp_comp_pool", mem_size, + shm = odp_shm_reserve("_odp_comp_global", mem_size, ODP_CACHE_LINE_SIZE, 0); global = odp_shm_addr(shm); diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index eb8085d40..bb40bc174 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -322,7 +322,7 @@ _odp_crypto_init_global(void) mem_size = sizeof(odp_crypto_global_t); /* Allocate our globally shared memory */ - shm = odp_shm_reserve("_odp_crypto_pool_null", mem_size, + shm = odp_shm_reserve("_odp_crypto_null_global", mem_size, ODP_CACHE_LINE_SIZE, 0); if (ODP_SHM_INVALID == shm) { @@ -359,7 +359,7 @@ int _odp_crypto_term_global(void) rc = -1; } - ret = odp_shm_free(odp_shm_lookup("_odp_crypto_pool_null")); + ret = odp_shm_free(odp_shm_lookup("_odp_crypto_null_global")); if (ret < 0) { ODP_ERR("shm free failed for _odp_crypto_pool_null\n"); rc = -1; diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 6cad38f92..3a649eae1 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -2530,7 +2530,7 @@ int _odp_crypto_init_global(void) mem_size += nlocks * sizeof(odp_ticketlock_t); /* Allocate our globally shared memory */ - shm = odp_shm_reserve("_odp_crypto_pool_ssl", mem_size, + shm = odp_shm_reserve("_odp_crypto_ssl_global", mem_size, ODP_CACHE_LINE_SIZE, 0); if (ODP_SHM_INVALID == shm) { @@ -2585,7 +2585,7 @@ int _odp_crypto_term_global(void) CRYPTO_set_id_callback(NULL); #endif - ret = odp_shm_free(odp_shm_lookup("_odp_crypto_pool_ssl")); + ret = odp_shm_free(odp_shm_lookup("_odp_crypto_ssl_global")); if (ret < 0) { ODP_ERR("shm free failed for crypto_pool\n"); rc = -1; diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 572c67747..1338f15e7 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -2241,7 +2241,7 @@ int _odp_ipsec_init_global(void) if (odp_global_ro.disable.ipsec) return 0; - shm = odp_shm_reserve("_odp_ipsec", sizeof(*ipsec_global), + shm = odp_shm_reserve("_odp_ipsec_global", sizeof(*ipsec_global), ODP_CACHE_LINE_SIZE, 0); if (shm == ODP_SHM_INVALID) { ODP_ERR("Shm reserve failed for odp_ipsec\n"); @@ -2277,7 +2277,7 @@ int _odp_ipsec_term_global(void) if (odp_global_ro.disable.ipsec) return 0; - shm = odp_shm_lookup("_odp_ipsec"); + shm = odp_shm_lookup("_odp_ipsec_global"); if (shm == ODP_SHM_INVALID || odp_shm_free(shm)) { ODP_ERR("Shm free failed for odp_ipsec"); diff --git a/platform/linux-generic/odp_pcapng.c b/platform/linux-generic/odp_pcapng.c index 68a18b305..ce950760e 100644 --- a/platform/linux-generic/odp_pcapng.c +++ b/platform/linux-generic/odp_pcapng.c @@ -90,7 +90,7 @@ int _odp_pcapng_init_global(void) { odp_shm_t shm; - shm = odp_shm_reserve("_odp_pcapng_gbl", sizeof(pcapng_global_t), + shm = odp_shm_reserve("_odp_pcapng_global", sizeof(pcapng_global_t), ODP_PAGE_SIZE, 0); if (shm == ODP_SHM_INVALID) return -1; diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index d1fc94369..9c60a9458 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -271,7 +271,7 @@ int _odp_pool_init_global(void) uint32_t i; odp_shm_t shm; - shm = odp_shm_reserve("_odp_pool_table", + shm = odp_shm_reserve("_odp_pool_global", sizeof(pool_global_t), ODP_CACHE_LINE_SIZE, 0); diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 76f897dfd..e35f01f7d 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -132,7 +132,7 @@ static int queue_init_global(void) _odp_queue_inline_offset.context = offsetof(queue_entry_t, s.param.context); - shm = odp_shm_reserve("_odp_queue_gbl", + shm = odp_shm_reserve("_odp_queue_basic_global", sizeof(queue_global_t), sizeof(queue_entry_t), 0); @@ -161,7 +161,7 @@ static int queue_init_global(void) mem_size = sizeof(uint32_t) * CONFIG_MAX_QUEUES * (uint64_t)_odp_queue_glb->config.max_queue_size; - shm = odp_shm_reserve("_odp_queue_rings", mem_size, + shm = odp_shm_reserve("_odp_queue_basic_rings", mem_size, ODP_CACHE_LINE_SIZE, 0); diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c index 70d555ab5..206172bb8 100644 --- a/platform/linux-generic/odp_queue_lf.c +++ b/platform/linux-generic/odp_queue_lf.c @@ -268,7 +268,7 @@ uint32_t _odp_queue_lf_init_global(uint32_t *queue_lf_size, if (!lockfree) return 0; - shm = odp_shm_reserve("_odp_queues_lf", sizeof(queue_lf_global_t), + shm = odp_shm_reserve("_odp_queues_lf_global", sizeof(queue_lf_global_t), ODP_CACHE_LINE_SIZE, 0); if (shm == ODP_SHM_INVALID) diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index ef0a880b6..eb17e95eb 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -463,7 +463,7 @@ static int schedule_init_global(void) ODP_DBG("Schedule init ... "); - shm = odp_shm_reserve("_odp_scheduler", + shm = odp_shm_reserve("_odp_sched_basic_global", sizeof(sched_global_t), ODP_CACHE_LINE_SIZE, 0); diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 18e738330..10f456eed 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -1792,7 +1792,7 @@ static int schedule_init_global(void) uint64_t min_alloc; uint64_t max_alloc; - shm = odp_shm_reserve("_odp_sched_scalable", + shm = odp_shm_reserve("_odp_sched_scalable_global", sizeof(sched_global_t), ODP_CACHE_LINE_SIZE, 0); diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index c4f8344f1..4f6ce736d 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -172,7 +172,7 @@ static int init_global(void) ODP_DBG("Using SP scheduler\n"); - shm = odp_shm_reserve("sp_scheduler", + shm = odp_shm_reserve("_odp_sched_sp_global", sizeof(sched_global_t), ODP_CACHE_LINE_SIZE, 0); diff --git a/platform/linux-generic/odp_stash.c b/platform/linux-generic/odp_stash.c index d2062379b..74b8df2de 100644 --- a/platform/linux-generic/odp_stash.c +++ b/platform/linux-generic/odp_stash.c @@ -57,7 +57,7 @@ int _odp_stash_init_global(void) { odp_shm_t shm; - shm = odp_shm_reserve("_odp_stash_table", sizeof(stash_global_t), + shm = odp_shm_reserve("_odp_stash_global", sizeof(stash_global_t), ODP_CACHE_LINE_SIZE, 0); stash_global = odp_shm_addr(shm); diff --git a/platform/linux-generic/odp_thread.c b/platform/linux-generic/odp_thread.c index af891bce8..ffb116974 100644 --- a/platform/linux-generic/odp_thread.c +++ b/platform/linux-generic/odp_thread.c @@ -68,7 +68,7 @@ int _odp_thread_init_global(void) if (num_max > ODP_THREAD_COUNT_MAX) num_max = ODP_THREAD_COUNT_MAX; - shm = odp_shm_reserve("_odp_thread_globals", + shm = odp_shm_reserve("_odp_thread_global", sizeof(thread_globals_t), ODP_CACHE_LINE_SIZE, 0); @@ -97,7 +97,7 @@ int _odp_thread_term_global(void) if (num) ODP_ERR("%u threads have not called odp_term_local().\n", num); - ret = odp_shm_free(odp_shm_lookup("_odp_thread_globals")); + ret = odp_shm_free(odp_shm_lookup("_odp_thread_global")); if (ret < 0) ODP_ERR("shm free failed for _odp_thread_globals"); diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index e5b42a83b..f1ca28346 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -1599,7 +1599,7 @@ int _odp_timer_init_global(const odp_init_t *params) return 0; } - shm = odp_shm_reserve("_odp_timer", sizeof(timer_global_t), + shm = odp_shm_reserve("_odp_timer_global", sizeof(timer_global_t), ODP_CACHE_LINE_SIZE, 0); timer_global = odp_shm_addr(shm); diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index 819204a41..e741fd80c 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -4936,7 +4936,7 @@ int _odp_tm_init_global(void) return 0; } - shm = odp_shm_reserve("_odp_traffic_mng", sizeof(tm_global_t), 0, 0); + shm = odp_shm_reserve("_odp_traffic_mng_global", sizeof(tm_global_t), 0, 0); if (shm == ODP_SHM_INVALID) return -1; -- cgit v1.2.3 From c6339d406ebb4fb525b47e1456c49e1773c65bad Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 20 Sep 2021 13:05:28 +0300 Subject: linux-gen: shm: allocate enough resources for internal allocations Reserve enough SHM blocks for implementation internal usage and return correct value in odp_shm_capability(). Removed ODP_ prefix from CONFIG_SHM_BLOCKS config header option. Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- platform/linux-generic/include/odp_config_internal.h | 20 +++++++++++++++----- platform/linux-generic/odp_ishm.c | 2 +- platform/linux-generic/odp_shared_memory.c | 5 +---- platform/linux-generic/odp_system_info.c | 2 +- 4 files changed, 18 insertions(+), 11 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 1e8b390dd..40d2639f1 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -20,9 +20,9 @@ extern "C" { #define CONFIG_NUM_CPU_IDS 256 /* - * Maximum number of pools. Limited by ISHM_MAX_NB_BLOCKS. + * Maximum number of pools. */ -#define ODP_CONFIG_POOLS 55 +#define ODP_CONFIG_POOLS 32 /* * Queues reserved for ODP internal use @@ -116,11 +116,21 @@ extern "C" { CONFIG_PACKET_HEADROOM + \ CONFIG_PACKET_TAILROOM) -/* Maximum number of shared memory blocks. +/* + * Number of shared memory blocks reserved for implementation internal use. + * + * Each pool requires three SHM blocks (buffers, ring, user area). 20 blocks are + * reserved for per ODP module global data. + */ +#define CONFIG_INTERNAL_SHM_BLOCKS ((ODP_CONFIG_POOLS * 3) + 20) + +/* + * Maximum number of shared memory blocks. * - * This the the number of separate SHM areas that can be reserved concurrently + * This is the number of separate SHM blocks that an application can reserve + * concurrently. */ -#define ODP_CONFIG_SHM_BLOCKS (ODP_CONFIG_POOLS + 48) +#define CONFIG_SHM_BLOCKS 64 /* * Maximum event burst size diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 7041cd2b5..ebc931658 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -75,7 +75,7 @@ * if some of the block owners never procsync() after free). This number * should take that into account) */ -#define ISHM_MAX_NB_BLOCKS 128 +#define ISHM_MAX_NB_BLOCKS (CONFIG_INTERNAL_SHM_BLOCKS + CONFIG_SHM_BLOCKS) /* * Maximum internal shared memory block name length in chars diff --git a/platform/linux-generic/odp_shared_memory.c b/platform/linux-generic/odp_shared_memory.c index ee47b7e96..966850a07 100644 --- a/platform/linux-generic/odp_shared_memory.c +++ b/platform/linux-generic/odp_shared_memory.c @@ -15,9 +15,6 @@ #include #include -ODP_STATIC_ASSERT(ODP_CONFIG_SHM_BLOCKS >= ODP_CONFIG_POOLS, - "ODP_CONFIG_SHM_BLOCKS < ODP_CONFIG_POOLS"); - static inline uint32_t from_handle(odp_shm_t shm) { return _odp_typeval(shm) - 1; @@ -47,7 +44,7 @@ int odp_shm_capability(odp_shm_capability_t *capa) { memset(capa, 0, sizeof(odp_shm_capability_t)); - capa->max_blocks = ODP_CONFIG_SHM_BLOCKS; + capa->max_blocks = CONFIG_SHM_BLOCKS; capa->max_size = odp_global_ro.shm_max_size; capa->max_align = 0; diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index 27c591a9c..778ea08cb 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -579,7 +579,7 @@ void odp_sys_config_print(void) ODP_PRINT("ODP_CONFIG_PKTIO_ENTRIES: %i\n", ODP_CONFIG_PKTIO_ENTRIES); ODP_PRINT("CONFIG_PACKET_HEADROOM: %i\n", CONFIG_PACKET_HEADROOM); ODP_PRINT("CONFIG_PACKET_TAILROOM: %i\n", CONFIG_PACKET_TAILROOM); - ODP_PRINT("ODP_CONFIG_SHM_BLOCKS: %i\n", ODP_CONFIG_SHM_BLOCKS); + ODP_PRINT("CONFIG_SHM_BLOCKS: %i\n", CONFIG_SHM_BLOCKS); ODP_PRINT("CONFIG_BURST_SIZE: %i\n", CONFIG_BURST_SIZE); ODP_PRINT("CONFIG_POOL_MAX_NUM: %i\n", CONFIG_POOL_MAX_NUM); ODP_PRINT("CONFIG_POOL_CACHE_MAX_SIZE: %i\n", CONFIG_POOL_CACHE_MAX_SIZE); -- cgit v1.2.3 From 423c841b7270118683badd395da1912a74a19e09 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 21 Sep 2021 09:18:50 +0300 Subject: linux-gen: crypto: null: add support for disabling crypto feature Add support for disabling null crypto using odp_init_t.not_used.feat.crypto option. Saves one SHM reservation. Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- platform/linux-generic/odp_crypto_null.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'platform') diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index bb40bc174..aa8e8a2b3 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -1,4 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -13,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -113,6 +115,11 @@ void free_session(odp_crypto_generic_session_t *session) int odp_crypto_capability(odp_crypto_capability_t *capa) { + if (odp_global_ro.disable.crypto) { + ODP_ERR("Crypto is disabled\n"); + return -1; + } + if (NULL == capa) return -1; @@ -190,6 +197,15 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, int rc; odp_crypto_generic_session_t *session; + if (odp_global_ro.disable.crypto) { + ODP_ERR("Crypto is disabled\n"); + /* Dummy output to avoid compiler warning about uninitialized + * variables */ + *status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + /* Allocate memory for this session */ session = alloc_session(); if (NULL == session) { @@ -318,6 +334,11 @@ _odp_crypto_init_global(void) odp_shm_t shm; int idx; + if (odp_global_ro.disable.crypto) { + ODP_PRINT("\nODP crypto is DISABLED\n"); + return 0; + } + /* Calculate the memory size we need */ mem_size = sizeof(odp_crypto_global_t); @@ -352,6 +373,9 @@ int _odp_crypto_term_global(void) int count = 0; odp_crypto_generic_session_t *session; + if (odp_global_ro.disable.crypto) + return 0; + for (session = global->free; session != NULL; session = session->next) count++; if (count != MAX_SESSIONS) { -- cgit v1.2.3 From 2bc64857084e338ce9a3417177b537cdf0ca0aa0 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 21 Sep 2021 09:31:31 +0300 Subject: linux-gen: stash: add support for disabling stash feature Add support for disabling stash using odp_init_t.not_used.feat.stash option. Saves one SHM reservation. Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- platform/linux-generic/include/odp_global_data.h | 1 + platform/linux-generic/odp_init.c | 1 + platform/linux-generic/odp_stash.c | 21 ++++++++++++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) (limited to 'platform') diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index a907f87be..e4fd583a4 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -74,6 +74,7 @@ typedef struct odp_global_data_ro_t { uint8_t compress; uint8_t crypto; uint8_t ipsec; + uint8_t stash; uint8_t traffic_mngr; } disable; diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index 0cb60cdfa..d5530c1f1 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -66,6 +66,7 @@ static void disable_features(odp_global_data_ro_t *global_ro, if (disable_ipsec && disable_crypto) global_ro->disable.crypto = 1; + global_ro->disable.stash = init_param->not_used.feat.stash; global_ro->disable.traffic_mngr = init_param->not_used.feat.tm; global_ro->disable.compress = init_param->not_used.feat.compress; } diff --git a/platform/linux-generic/odp_stash.c b/platform/linux-generic/odp_stash.c index 74b8df2de..a1bf5b63d 100644 --- a/platform/linux-generic/odp_stash.c +++ b/platform/linux-generic/odp_stash.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2020, Nokia +/* Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -57,6 +58,11 @@ int _odp_stash_init_global(void) { odp_shm_t shm; + if (odp_global_ro.disable.stash) { + ODP_PRINT("Stash is DISABLED\n"); + return 0; + } + shm = odp_shm_reserve("_odp_stash_global", sizeof(stash_global_t), ODP_CACHE_LINE_SIZE, 0); @@ -76,6 +82,9 @@ int _odp_stash_init_global(void) int _odp_stash_term_global(void) { + if (odp_global_ro.disable.stash) + return 0; + if (stash_global == NULL) return 0; @@ -89,6 +98,11 @@ int _odp_stash_term_global(void) int odp_stash_capability(odp_stash_capability_t *capa, odp_stash_type_t type) { + if (odp_global_ro.disable.stash) { + ODP_ERR("Stash is disabled\n"); + return -1; + } + (void)type; memset(capa, 0, sizeof(odp_stash_capability_t)); @@ -146,6 +160,11 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) int ring_ptr, index; char shm_name[ODP_STASH_NAME_LEN + 8]; + if (odp_global_ro.disable.stash) { + ODP_ERR("Stash is disabled\n"); + return ODP_STASH_INVALID; + } + if (param->obj_size > sizeof(uintptr_t)) { ODP_ERR("Too large object handle.\n"); return ODP_STASH_INVALID; -- cgit v1.2.3 From 61744dc3debd1db8974e5d622c97c0a4a0039947 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 27 Aug 2021 15:03:44 +0300 Subject: linux-gen: crypto: fix AES-XCBC-MAC calculation with segmented packets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a segment boundary of a multisegment packet is not at AES block boundary, a partial block's worth of data is carried over to the processing of the next segment. When the partial block is then filled with data from that segment, the data pointer is not updated, i.e. the data is not considered used. This causes the next block consume the same packet data again, resulting in an incorrect digest. Fix the problem by adding the missing data pointer update. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen --- platform/linux-generic/odp_crypto_openssl.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 3a649eae1..c22d3603d 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -438,8 +438,12 @@ void packet_aes_xcbc_mac(odp_packet_t pkt, len -= datalen; if (eoff != 0) { if (eoff + datalen > AES_BLOCK_SIZE) { - memxor(e + eoff, data, AES_BLOCK_SIZE - eoff); - datalen -= (AES_BLOCK_SIZE - eoff); + /* bytes needed to fill the partial block */ + uint32_t remaining_len = AES_BLOCK_SIZE - eoff; + + memxor(e + eoff, data, remaining_len); + datalen -= remaining_len; + data += remaining_len; eoff = 0; EVP_EncryptUpdate(ctx, e, &dummy_len, e, sizeof(e)); -- cgit v1.2.3 From 92c26224d96266e3dfba5836aa8cddd07fbd08fa Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 27 Aug 2021 15:03:50 +0300 Subject: linux-gen: crypto: fix encryption & decryption of segmented packets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrite internal_encrypt() and internal_decrypt() to handle segmented packets properly. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen --- platform/linux-generic/odp_crypto_openssl.c | 233 ++++++++++++++++------------ 1 file changed, 130 insertions(+), 103 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index c22d3603d..9e5327109 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -1,4 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -898,120 +899,146 @@ int internal_aad(EVP_CIPHER_CTX *ctx, return ret; } -static -int internal_encrypt(EVP_CIPHER_CTX *ctx, - odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param) -{ - unsigned in_pos = param->cipher_range.offset; - unsigned out_pos = param->cipher_range.offset; - unsigned in_len = param->cipher_range.length; - uint8_t block[2 * EVP_MAX_BLOCK_LENGTH]; - unsigned block_len = EVP_CIPHER_block_size(EVP_CIPHER_CTX_cipher(ctx)); - int cipher_len; - int ret; - - ODP_ASSERT(in_pos + in_len <= odp_packet_len(pkt)); - - while (in_len > 0) { - uint32_t seglen = 0; /* GCC */ - uint8_t *insegaddr = odp_packet_offset(pkt, in_pos, - &seglen, NULL); - unsigned inseglen = in_len < seglen ? in_len : seglen; - - /* There should be at least 1 additional block in out buffer */ - if (inseglen > block_len) { - unsigned part = inseglen - block_len; - - EVP_EncryptUpdate(ctx, insegaddr, &cipher_len, - insegaddr, part); - in_pos += part; - in_len -= part; - insegaddr += part; - inseglen -= part; - - out_pos += cipher_len; - } +typedef int (*evp_update_t)(EVP_CIPHER_CTX *, unsigned char *, + int *, const unsigned char *, int); - /* Use temporal storage */ - if (inseglen > 0) { - unsigned part = inseglen; +typedef int (*evp_final_t)(EVP_CIPHER_CTX *, unsigned char *, int *); - EVP_EncryptUpdate(ctx, block, &cipher_len, - insegaddr, part); - in_pos += part; - in_len -= part; - insegaddr += part; - inseglen -= part; - - odp_packet_copy_from_mem(pkt, out_pos, - cipher_len, block); - out_pos += cipher_len; - } - } - - ret = EVP_EncryptFinal_ex(ctx, block, &cipher_len); - odp_packet_copy_from_mem(pkt, out_pos, cipher_len, block); - - return ret; -} - -static -int internal_decrypt(EVP_CIPHER_CTX *ctx, - odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param) -{ - unsigned in_pos = param->cipher_range.offset; - unsigned out_pos = param->cipher_range.offset; - unsigned in_len = param->cipher_range.length; - uint8_t block[2 * EVP_MAX_BLOCK_LENGTH]; - unsigned block_len = EVP_CIPHER_block_size(EVP_CIPHER_CTX_cipher(ctx)); - int cipher_len; - int ret; +static inline int internal_crypt(EVP_CIPHER_CTX *ctx, + odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + evp_update_t EVP_update, + evp_final_t EVP_final) +{ + uint32_t in_pos = param->cipher_range.offset; + uint32_t out_pos = in_pos; + uint32_t in_len = param->cipher_range.length; + uint8_t block[EVP_MAX_BLOCK_LENGTH]; + uint32_t buffered = 0; + uint32_t block_len = EVP_CIPHER_block_size(EVP_CIPHER_CTX_cipher(ctx)); + int out_len; + int rc; ODP_ASSERT(in_pos + in_len <= odp_packet_len(pkt)); + /* + * In the following loop we process one packet segment per iteration. + * We rely on the following properties of the encrypt/decrypt update + * function with the algorithms that we use: + * + * - The function processes (and writes to output) only whole blocks. + * - Input data beyond the last full block is buffered inside OpenSSL. + * - The amount of buffered data is always less than one block. + * - Total amount of output data does not exceed the total amount + * of input data at any point. + */ while (in_len > 0) { - uint32_t seglen = 0; /* GCC */ - uint8_t *insegaddr = odp_packet_offset(pkt, in_pos, - &seglen, NULL); - unsigned inseglen = in_len < seglen ? in_len : seglen; - - /* There should be at least 1 additional block in out buffer */ - if (inseglen > block_len) { - unsigned part = inseglen - block_len; - - EVP_DecryptUpdate(ctx, insegaddr, &cipher_len, - insegaddr, part); - in_pos += part; - in_len -= part; - insegaddr += part; - inseglen -= part; - - out_pos += cipher_len; + uint32_t seglen = 0; + uint8_t *in_addr = odp_packet_offset(pkt, in_pos, + &seglen, NULL); + uint32_t len = in_len < seglen ? in_len : seglen; + + if (odp_unlikely(buffered > 0)) { + /* + * Leftover data from the previous segment is + * in the buffer inside OpenSSL. + */ + uint32_t remaining_len = block_len - buffered; + + if (odp_likely(len >= remaining_len)) { + /* + * Let's fill the buffered input data to a + * full block and get the output block to + * a memory buffer. The buffer is then copied + * to the packet, crossing segment boundary. + */ + rc = EVP_update(ctx, block, &out_len, + in_addr, remaining_len); + if (odp_unlikely(rc != 1)) + goto err; + if (odp_unlikely(out_len != (int)block_len)) + goto err; + in_addr += remaining_len; + in_pos += remaining_len; + len -= remaining_len; + in_len -= remaining_len; + buffered = 0; + rc = odp_packet_copy_from_mem(pkt, out_pos, + block_len, block); + if (odp_unlikely(rc)) + goto err; + out_pos += block_len; + } else { + /* + * Not enough data in this segment to fill + * the buffer to a full block. Fill the buffer + * a bit more and go to the next segment. + */ + rc = EVP_update(ctx, block, &out_len, + in_addr, len); + if (odp_unlikely(rc != 1)) + goto err; + if (odp_unlikely(out_len > 0)) + goto err; + in_pos += len; + in_len -= len; + buffered += len; + continue; + } } - - /* Use temporal storage */ - if (inseglen > 0) { - unsigned part = inseglen; - - EVP_DecryptUpdate(ctx, block, &cipher_len, - insegaddr, part); - in_pos += part; - in_len -= part; - insegaddr += part; - inseglen -= part; - - odp_packet_copy_from_mem(pkt, out_pos, - cipher_len, block); - out_pos += cipher_len; + ODP_ASSERT(buffered == 0); + + if (in_len > 0) { + /* + * No input is buffered inside OpenSSL. We pass the + * whole remaining segment to OpenSSL and expect to + * get a multiple of block size of data processed, + * with the rest left in the buffer. + */ + rc = EVP_update(ctx, in_addr, &out_len, in_addr, len); + if (odp_unlikely(rc != 1)) + goto err; + ODP_ASSERT(CHECK_IS_POWER2(block_len)); + buffered = len & (block_len - 1); + if (odp_unlikely(out_len + buffered != len)) + goto err; + in_pos += len; + in_len -= len; + out_pos += len - buffered; } } + if (odp_unlikely(buffered > 0)) + goto err; + /* + * We do not expect any more data out since the cipher range is + * supposed to be a multiple of the block size. + */ + rc = EVP_final(ctx, block, &out_len); + if (odp_unlikely(out_len != 0)) + return 0; + return rc; +err: + ODP_ERR("internal error\n"); + (void)EVP_final(ctx, block, &out_len); + return 0; +} - ret = EVP_DecryptFinal_ex(ctx, block, &cipher_len); - odp_packet_copy_from_mem(pkt, out_pos, cipher_len, block); +static int internal_encrypt(EVP_CIPHER_CTX *ctx, + odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param) +{ + return internal_crypt(ctx, pkt, param, + EVP_EncryptUpdate, + EVP_EncryptFinal_ex); +} - return ret; +static int internal_decrypt(EVP_CIPHER_CTX *ctx, + odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param) +{ + return internal_crypt(ctx, pkt, param, + EVP_DecryptUpdate, + EVP_DecryptFinal_ex); } static void -- cgit v1.2.3 From ad9638fd922d8a7735787cbd1870ace6c7593738 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Mon, 13 Sep 2021 13:50:17 +0300 Subject: linux-gen: packet: fix odp_packet_parse() with non-zero offset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When odp_packet_parse() parses an ethernet frame at non-zero offset, normal length Ethernet frames may get misinterpreted as jumbo frames. This is because parse_eth() fails to subtract the offset from the total ODP packet length when checking the Ethernet frame length. Fix the problem by subtracting the offset. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen --- platform/linux-generic/odp_packet.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 0986056e6..da59725f8 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1864,7 +1864,7 @@ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, eth = (const _odp_ethhdr_t *)*parseptr; /* Detect jumbo frames */ - if (odp_unlikely(frame_len > _ODP_ETH_LEN_MAX)) + if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX)) input_flags.jumbo = 1; /* Handle Ethernet broadcast/multicast addresses */ -- cgit v1.2.3 From 50fe59300246fc37da655ae9b74bcf3b04fb8927 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Mon, 13 Sep 2021 13:50:27 +0300 Subject: linux-gen: packet: fix Ethernet frame length checking in parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check that Ethernet frame length is long enough to contain all the fields that got parsed and fail Ethernet parsing if not. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen --- platform/linux-generic/odp_packet.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'platform') diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index da59725f8..416af3960 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1922,6 +1922,16 @@ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_vlanhdr_t); } + /* + * The packet was too short for what we parsed. We just give up + * entirely without trying to parse what fits in the packet. + */ + if (odp_unlikely(*offset > frame_len)) { + input_flags.all = 0; + input_flags.l2 = 1; + ethtype = 0; + } + error: prs->input_flags.all |= input_flags.all; -- cgit v1.2.3 From adae181a7f5a2ecb693bd37235d2019a46e55b7c Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Mon, 13 Sep 2021 13:50:31 +0300 Subject: linux-gen: packet: fix packet length checking in IP parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When parsing IPv4 or IPv6, check that there is enough packet data for the IP header. Checking the length fields of the header is not sufficient if the length fields may be read from beyond the end of the packet data and accidentally look ok. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen --- platform/linux-generic/odp_packet.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 416af3960..1e0effb59 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1955,6 +1955,7 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN || ver != 4 || + sizeof(*ipv4) > frame_len - *offset || (l3_len > frame_len - *offset))) { prs->flags.ip_err = 1; return 0; @@ -2013,8 +2014,9 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, _ODP_IPV6HDR_LEN; /* Basic sanity checks on IPv6 header */ - if ((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || - l3_len > frame_len - *offset) { + if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || + sizeof(*ipv6) > frame_len - *offset || + l3_len > frame_len - *offset)) { prs->flags.ip_err = 1; return 0; } -- cgit v1.2.3 From b70771a1d5774fc00211a0a4edfcfff76489d067 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Mon, 13 Sep 2021 13:50:35 +0300 Subject: linux-gen: packet: fix SCTP checksum check with multi-segment packets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit packet_l4_chksum() does not take possible packet segmentation into account when checking the checksum value of the SCTP header and may end up reading the checksum field from an incorrect memory location. This requires packet segment boundary occurring in the middle of the SCTP header. That is possible because the function is used by odp_packet_parse() that parses user provided packets at user provided data offset. Fix the problem by using odp_packet_copy_to_mem() to copy the SCTP header into a temporary buffer if needed. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen --- platform/linux-generic/odp_packet.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 1e0effb59..7aacaa6c0 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -2528,6 +2528,8 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, if (chksums.chksum.sctp && pkt_hdr->p.input_flags.sctp && !pkt_hdr->p.input_flags.ipfrag) { + uint32_t seg_len = 0; + _odp_sctphdr_t hdr_copy; uint32_t sum = ~packet_sum_crc32c(pkt_hdr, pkt_hdr->p.l4_offset + _ODP_SCTPHDR_LEN, @@ -2537,8 +2539,14 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, l4_part_sum); _odp_sctphdr_t *sctp = packet_map(pkt_hdr, pkt_hdr->p.l4_offset, - NULL, NULL); + &seg_len, NULL); + if (odp_unlikely(seg_len < sizeof(*sctp))) { + odp_packet_t pkt = packet_handle(pkt_hdr); + sctp = &hdr_copy; + odp_packet_copy_to_mem(pkt, pkt_hdr->p.l4_offset, + sizeof(*sctp), sctp); + } pkt_hdr->p.input_flags.l4_chksum_done = 1; if (sum != sctp->chksum) { pkt_hdr->p.flags.l4_chksum_err = 1; -- cgit v1.2.3 From 6b93469a77232031113e63616d124dfe41627e29 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Mon, 13 Sep 2021 13:50:38 +0300 Subject: linux-gen: packet: fix parsing of segmented packets in odp_packet_parse() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit odp_packet_parse() cannot assume anything about the segmentation of the parsed packet that is provided by an ODP application and cannot thus directly pass packet pointers to parsing functions that work on contiguous data. Fix the problem by copying enough packet data into a temporary buffer if necessary (i.e. if the packet is a multi-segment packet with a segment boundary in the potentially parsed region). Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen --- platform/linux-generic/odp_packet.c | 54 ++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 7aacaa6c0..d14ee5b68 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -1847,7 +1848,15 @@ static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr, return sum; } -/** Parser helper function for Ethernet packets */ +/* + * In the worst case we look at the Ethernet header, 8 bytes of LLC/SNAP + * header and two VLAN tags in the same packet. + */ +#define PARSE_ETH_BYTES (sizeof(_odp_ethhdr_t) + 8 + 2 * sizeof(_odp_vlanhdr_t)) +/** Parser helper function for Ethernet packets + * + * Requires up to PARSE_ETH_BYTES bytes of contiguous packet data. + */ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len) { @@ -1938,8 +1947,11 @@ error: return ethtype; } +#define PARSE_IPV4_BYTES (0xfU * 4) /* max IPv4 header length with options */ /** * Parser helper function for IPv4 + * + * Requires up to PARSE_IPV4_BYTES bytes of contiguous packet data. */ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, @@ -1998,8 +2010,15 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, return ipv4->proto; } +/* + * Peeks 2 bytes beyond IPv6 base header without length check if there + * are extension headers. + */ +#define PARSE_IPV6_BYTES (sizeof(_odp_ipv6hdr_t) + 2) /** * Parser helper function for IPv6 + * + * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data. */ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, @@ -2068,8 +2087,11 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, return ipv6->next_hdr; } +#define PARSE_TCP_BYTES (sizeof(_odp_tcphdr_t)) /** * Parser helper function for TCP + * + * Requires PARSE_TCP_BYTES bytes of contiguous packet data. */ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t tcp_len, @@ -2095,8 +2117,15 @@ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += len; } +/* + * In the worst case we look at the UDP header and 4 bytes of the UDP + * payload (the non-ESP marker to distinguish IKE packets from ESP packets). + */ +#define PARSE_UDP_BYTES (sizeof(_odp_udphdr_t) + 4) /** * Parser helper function for UDP + * + * Requires PARSE_UDP_BYTES bytes of contiguous packet data. */ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, odp_proto_chksums_t chksums, @@ -2141,8 +2170,11 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_udphdr_t); } +#define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t)) /** * Parser helper function for SCTP + * + * Requires PARSE_SCTP_BYTES bytes of contiguous packet data. */ static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t sctp_len, @@ -2169,6 +2201,10 @@ static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_sctphdr_t); } +#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c))) +#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \ + MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES)) +/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */ static inline int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, uint32_t offset, @@ -2607,6 +2643,8 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, int ret; uint16_t ethtype; uint64_t l4_part_sum = 0; + const uint32_t min_seglen = PARSE_ETH_BYTES + PARSE_L3_L4_BYTES; + uint8_t buf[min_seglen]; if (proto == ODP_PROTO_NONE || layer == ODP_PROTO_LAYER_NONE) return -1; @@ -2616,6 +2654,20 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, if (data == NULL) return -1; + /* + * We must not have a packet segment boundary within the parsed + * packet data range. Copy enough data to a temporary buffer for + * parsing if necessary. + */ + if (odp_unlikely(pkt_hdr->seg_count > 1) && + odp_unlikely(seg_len < min_seglen)) { + seg_len = min_seglen; + if (seg_len > packet_len - offset) + seg_len = packet_len - offset; + odp_packet_copy_to_mem(pkt, offset, seg_len, buf); + data = buf; + } + /* Reset parser flags, keep other flags */ packet_parse_reset(pkt_hdr, 0); -- cgit v1.2.3 From e108b53c8d3138151257bebfd918305cc7479b73 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Mon, 13 Sep 2021 13:50:41 +0300 Subject: linux-gen: pktio: loop: fix parsing of segmented packets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The receive processing in loop pktio processes packets that have the same underlying packet buffers as they had when sent. This means that, unlike other pktios that allocate packet buffers, loop may have packets with inconvenient segment boundaries in the receive path. Such packets may get parsed incorrectly since loop uses a parsing function that assumes contiguous packet data for the parsed headers. Fix the problem by doing parsing using odp_packet_parse() which can handle segmented packets too. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen --- platform/linux-generic/pktio/loop.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index 437977771..889a270ea 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -172,9 +172,16 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pkt_hdr = packet_hdr(new_pkt); } } else { - _odp_packet_parse_layer(pkt_hdr, - pktio_entry->s.config.parser.layer, - pktio_entry->s.in_chksums); + odp_packet_parse_param_t param; + + /* + * Use odp_packet_parse() which can handle segmented + * packets. + */ + param.proto = ODP_PROTO_ETH; + param.last_layer = pktio_entry->s.config.parser.layer; + param.chksums = pktio_entry->s.in_chksums; + odp_packet_parse(packet_handle(pkt_hdr), 0, ¶m); } packet_set_ts(pkt_hdr, ts); -- cgit v1.2.3 From f9c411151743ee11200483812757b55475d591b1 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 23 Sep 2021 11:25:46 +0300 Subject: crypto: fix crypto and auth range offset in bit mode In bit mode the crypto and auth ranges are supposed to be given in bits, not in bytes. Fix the linux-gen implementation that interpreted crypto and auth offset in bytes in AES EEA2 and AES EIA2, respectively. Fix crypto validation test to provide the offsets in bits as specified by the API. Signed-off-by: Janne Peltonen Reviewed-by: Matias Elo Reviewed-by: Anoob Joseph --- platform/linux-generic/odp_crypto_openssl.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 9e5327109..d8276b4b4 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -700,7 +700,7 @@ int packet_cmac_eia2(odp_packet_t pkt, { CMAC_CTX *ctx = local.cmac_ctx[session->idx]; void *iv_ptr; - uint32_t offset = param->auth_range.offset; + uint32_t offset = param->auth_range.offset / 8; uint32_t len = (param->auth_range.length + 7) / 8; size_t outlen; @@ -1152,6 +1152,10 @@ odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt, uint32_t in_len = (param->cipher_range.length + 7) / 8; uint8_t data[in_len]; int ret; + uint32_t offset; + + /* Range offset is in bits in bit mode but must be divisible by 8. */ + offset = param->cipher_range.offset / 8; if (param->cipher_iv_ptr) iv_ptr = param->cipher_iv_ptr; @@ -1162,16 +1166,14 @@ odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt, EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - odp_packet_copy_to_mem(pkt, param->cipher_range.offset, in_len, - data); + odp_packet_copy_to_mem(pkt, offset, in_len, data); EVP_EncryptUpdate(ctx, data, &cipher_len, data, in_len); ret = EVP_EncryptFinal_ex(ctx, data + cipher_len, &dummy_len); cipher_len += dummy_len; - odp_packet_copy_from_mem(pkt, param->cipher_range.offset, in_len, - data); + odp_packet_copy_from_mem(pkt, offset, in_len, data); return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; @@ -1190,6 +1192,10 @@ odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt, uint32_t in_len = (param->cipher_range.length + 7) / 8; uint8_t data[in_len]; int ret; + uint32_t offset; + + /* Range offset is in bits in bit mode but must be divisible by 8. */ + offset = param->cipher_range.offset / 8; if (param->cipher_iv_ptr) iv_ptr = param->cipher_iv_ptr; @@ -1200,16 +1206,14 @@ odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt, EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - odp_packet_copy_to_mem(pkt, param->cipher_range.offset, in_len, - data); + odp_packet_copy_to_mem(pkt, offset, in_len, data); EVP_DecryptUpdate(ctx, data, &cipher_len, data, in_len); ret = EVP_DecryptFinal_ex(ctx, data + cipher_len, &dummy_len); cipher_len += dummy_len; - odp_packet_copy_from_mem(pkt, param->cipher_range.offset, in_len, - data); + odp_packet_copy_from_mem(pkt, offset, in_len, data); return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; -- cgit v1.2.3 From 3852994dfe0f6691ba5ccc28b47e3598853eaa6d Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Mon, 20 Sep 2021 13:30:36 +0300 Subject: linux-gen: ipsec: fix handling of completion event enqueueing failure If an IPsec-processed packet cannot be enqueued to the destination queue, indicate the packet as consumed to the caller and free the packet. Since the input packet has already been modified, it is not possible to report it as not consumed. Previously the packet was both freed and reported as not consumed. Not generating a completion event for an async IPsec operation may not be fully compliant with the current API spec, but it is probably better than mangling or freeing a packet that is reported as not consumed. Fix also the bug where inbound SA was not 'unused' after processing if the destination queue enqueuing failed. Signed-off-by: Janne Peltonen Reviewed-by: Petri Savolainen --- platform/linux-generic/odp_ipsec.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 1338f15e7..f669b5b29 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -1855,6 +1855,7 @@ int odp_ipsec_in_enq(const odp_packet_t pkt_in[], int num_in, ipsec_sa_t *ipsec_sa; odp_ipsec_packet_result_t *result; odp_queue_t queue; + int rc; memset(&status, 0, sizeof(status)); @@ -1879,10 +1880,6 @@ int odp_ipsec_in_enq(const odp_packet_t pkt_in[], int num_in, queue = ipsec_config->inbound.default_queue; } - if (odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt))) { - odp_packet_free(pkt); - break; - } in_pkt++; sa_idx += sa_inc; @@ -1893,6 +1890,12 @@ int odp_ipsec_in_enq(const odp_packet_t pkt_in[], int num_in, */ if (sa == ODP_IPSEC_SA_INVALID && ipsec_sa) _odp_ipsec_sa_unuse(ipsec_sa); + + rc = odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt)); + if (odp_unlikely(rc)) { + odp_packet_free(pkt); + break; + } } return in_pkt; @@ -1917,6 +1920,7 @@ int odp_ipsec_out_enq(const odp_packet_t pkt_in[], int num_in, odp_ipsec_packet_result_t *result; const odp_ipsec_out_opt_t *opt; odp_queue_t queue; + int rc; memset(&status, 0, sizeof(status)); @@ -1941,13 +1945,15 @@ int odp_ipsec_out_enq(const odp_packet_t pkt_in[], int num_in, if (ipsec_config->stats_en) ipsec_sa_err_stats_update(ipsec_sa, &status); - if (odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt))) { - odp_packet_free(pkt); - break; - } in_pkt++; sa_idx += sa_inc; opt_idx += opt_inc; + + rc = odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt)); + if (odp_unlikely(rc)) { + odp_packet_free(pkt); + break; + } } return in_pkt; @@ -2098,6 +2104,7 @@ int odp_ipsec_out_inline(const odp_packet_t pkt_in[], int num_in, } } else { odp_queue_t queue; + int rc; err: if (ipsec_config->stats_en) ipsec_sa_err_stats_update(ipsec_sa, &status); @@ -2109,8 +2116,9 @@ err: result->status = status; queue = ipsec_sa->queue; - if (odp_queue_enq(queue, - odp_ipsec_packet_to_event(pkt))) { + rc = odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt)); + if (odp_unlikely(rc)) { + in_pkt++; odp_packet_free(pkt); break; } -- cgit v1.2.3 From 98d8b6cb7cc5a2dd45838679a9d5e3e26327d113 Mon Sep 17 00:00:00 2001 From: Aakash Sasidharan Date: Wed, 7 Apr 2021 10:41:00 +0000 Subject: linux-gen: ipsec: set destination queue capability Set the destination queue capability. Signed-off-by: Aakash Sasidharan Reviewed-by: Janne Peltonen --- platform/linux-generic/odp_ipsec.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'platform') diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index f669b5b29..2bf9c7343 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -165,6 +165,9 @@ int odp_ipsec_capability(odp_ipsec_capability_t *capa) if (rc < 0) return rc; + capa->queue_type_plain = true; + capa->queue_type_sched = true; + rc = odp_queue_capability(&queue_capa); if (rc < 0) return rc; -- cgit v1.2.3 From 931e5cbdddd3a9a61f7e9afa373841ae947aede2 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 9 Sep 2021 14:53:46 +0300 Subject: linux-gen: ipsec: provide original packet length in IPsec packet result Provide the original ESP/AH packet length in IPsec packet result as specified in the API. Signed-off-by: Janne Peltonen Reviewed-by: Petri Savolainen --- platform/linux-generic/odp_ipsec.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 2bf9c7343..09596b502 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -760,7 +760,8 @@ static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, odp_ipsec_sa_t sa, odp_packet_t *pkt_out, odp_bool_t enqueue_op, - odp_ipsec_op_status_t *status) + odp_ipsec_op_status_t *status, + uint32_t *orig_ip_len) { ipsec_state_t state; ipsec_sa_t *ipsec_sa = NULL; @@ -796,6 +797,7 @@ static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, status->error.alg = 1; goto exit; } + *orig_ip_len = state.ip_tot_len; /* Check IP header for IPSec protocols and look it up */ if (_ODP_IPPROTO_ESP == state.ip_next_hdr || @@ -1745,6 +1747,7 @@ int odp_ipsec_in(const odp_packet_t pkt_in[], int num_in, odp_ipsec_op_status_t status; odp_ipsec_sa_t sa; ipsec_sa_t *ipsec_sa; + uint32_t dummy; /* orig_ip_len not valid in sync operations */ odp_ipsec_packet_result_t *result; memset(&status, 0, sizeof(status)); @@ -1756,7 +1759,7 @@ int odp_ipsec_in(const odp_packet_t pkt_in[], int num_in, ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); } - ipsec_sa = ipsec_in_single(pkt, sa, &pkt, false, &status); + ipsec_sa = ipsec_in_single(pkt, sa, &pkt, false, &status, &dummy); packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); result = ipsec_pkt_result(pkt); @@ -1856,6 +1859,7 @@ int odp_ipsec_in_enq(const odp_packet_t pkt_in[], int num_in, odp_ipsec_op_status_t status; odp_ipsec_sa_t sa; ipsec_sa_t *ipsec_sa; + uint32_t orig_ip_len = 0; odp_ipsec_packet_result_t *result; odp_queue_t queue; int rc; @@ -1869,12 +1873,13 @@ int odp_ipsec_in_enq(const odp_packet_t pkt_in[], int num_in, ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); } - ipsec_sa = ipsec_in_single(pkt, sa, &pkt, true, &status); + ipsec_sa = ipsec_in_single(pkt, sa, &pkt, true, &status, &orig_ip_len); packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); result = ipsec_pkt_result(pkt); memset(result, 0, sizeof(*result)); result->status = status; + result->orig_ip_len = orig_ip_len; if (NULL != ipsec_sa) { result->sa = ipsec_sa->ipsec_sa_hdl; queue = ipsec_sa->queue; @@ -1966,6 +1971,7 @@ int _odp_ipsec_try_inline(odp_packet_t *pkt) { odp_ipsec_op_status_t status; ipsec_sa_t *ipsec_sa; + uint32_t orig_ip_len = 0; odp_ipsec_packet_result_t *result; odp_packet_hdr_t *pkt_hdr; @@ -1975,7 +1981,7 @@ int _odp_ipsec_try_inline(odp_packet_t *pkt) memset(&status, 0, sizeof(status)); ipsec_sa = ipsec_in_single(*pkt, ODP_IPSEC_SA_INVALID, pkt, false, - &status); + &status, &orig_ip_len); /* * Route packet back in case of lookup failure or early error before * lookup @@ -1987,6 +1993,7 @@ int _odp_ipsec_try_inline(odp_packet_t *pkt) result = ipsec_pkt_result(*pkt); memset(result, 0, sizeof(*result)); result->status = status; + result->orig_ip_len = orig_ip_len; result->sa = ipsec_sa->ipsec_sa_hdl; result->flag.inline_mode = 1; -- cgit v1.2.3 From bf4f2390db1972f84d8c426662195fda1ae9ee9b Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Mon, 30 Aug 2021 12:55:38 +0300 Subject: linux-gen: packet: implement minimal odp_packet_reass_info() Implement odp_packet_reass_info() that always fails since linux-gen does not support reassembly offload and thus cannot have packets that may be passed to odp_packet_reass_info(). Signed-off-by: Janne Peltonen Reviewed-by: Anoob Joseph Reviewed-by: Petri Savolainen --- platform/linux-generic/odp_packet.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'platform') diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index d14ee5b68..c6a50bf84 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -3014,6 +3014,13 @@ odp_packet_reass_status(odp_packet_t pkt) return ODP_PACKET_REASS_NONE; } +int odp_packet_reass_info(odp_packet_t pkt, odp_packet_reass_info_t *info) +{ + (void)pkt; + (void)info; + return -1; +} + int odp_packet_reass_partial_state(odp_packet_t pkt, odp_packet_t frags[], odp_packet_reass_partial_state_t *res) -- cgit v1.2.3 From a369bcd19a32cae230b9428e29c008dbd785c681 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 14 Sep 2021 12:16:50 +0300 Subject: linux-gen: cpu: refactor inlining code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor CPU ABI and implementation files to follow common project inlining style. odp_cpu_pause() function is now defined only once per architecture. After this clean-up it's simpler for an implementation to add inlined versions of CPU module functions. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- platform/linux-generic/Makefile.am | 32 +++++++------ .../linux-generic/arch/aarch64/odp/api/abi/cpu.h | 11 +---- .../arch/aarch64/odp/api/abi/cpu_inlines.h | 32 +++++++++++++ platform/linux-generic/arch/arm/odp/api/abi/cpu.h | 11 +---- .../arch/arm/odp/api/abi/cpu_inlines.h | 32 +++++++++++++ .../linux-generic/arch/default/odp/api/abi/cpu.h | 5 +- .../arch/default/odp/api/abi/cpu_generic.h | 34 +++++++++++++ .../arch/default/odp/api/abi/cpu_inlines.h | 17 +++++-- .../linux-generic/arch/default/odp_cpu_cycles.c | 21 ++++---- .../linux-generic/arch/mips64/odp/api/abi/cpu.h | 9 +--- .../arch/mips64/odp/api/abi/cpu_inlines.h | 56 ++++++++++++++++++++++ .../linux-generic/arch/mips64/odp_cpu_cycles.c | 37 -------------- .../linux-generic/arch/powerpc/odp/api/abi/cpu.h | 20 +++++++- .../arch/x86/odp/api/abi/cpu_inlines.h | 17 +++++-- .../include/odp/api/plat/cpu_inlines.h | 25 +++++++++- 15 files changed, 256 insertions(+), 103 deletions(-) create mode 100644 platform/linux-generic/arch/aarch64/odp/api/abi/cpu_inlines.h create mode 100644 platform/linux-generic/arch/arm/odp/api/abi/cpu_inlines.h create mode 100644 platform/linux-generic/arch/default/odp/api/abi/cpu_generic.h create mode 100644 platform/linux-generic/arch/mips64/odp/api/abi/cpu_inlines.h delete mode 100644 platform/linux-generic/arch/mips64/odp_cpu_cycles.c (limited to 'platform') diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 8c75e5ec0..e763c0abc 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -275,12 +275,13 @@ __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/arm/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/arm/odp/api/abi/cpu_inlines.h \ arch/arm/odp/api/abi/cpu.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ @@ -297,14 +298,15 @@ __LIB__libodp_linux_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/aarch64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/aarch64/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu_time.h \ arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/aarch64/odp/api/abi/atomic_inlines.h \ arch/aarch64/odp/api/abi/atomic.h \ - arch/aarch64/odp/api/abi/cpu.h + arch/default/odp/api/abi/cpu_generic.h \ + arch/aarch64/odp/api/abi/cpu_inlines.h \ + arch/aarch64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ @@ -317,12 +319,13 @@ __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -331,16 +334,16 @@ noinst_HEADERS += arch/default/odp_atomic.h \ endif if ARCH_IS_MIPS64 __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ - arch/mips64/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/mips64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/mips64/odp/api/abi/cpu_inlines.h \ arch/mips64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -353,12 +356,13 @@ __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/powerpc/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/default/odp/api/abi/cpu_inlines.h \ arch/powerpc/odp/api/abi/cpu.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -372,13 +376,13 @@ __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/x86/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_inlines.h \ - arch/x86/odp/api/abi/cpu_rdtsc.h \ +odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_rdtsc.h \ arch/x86/odp/api/abi/cpu_time.h \ arch/x86/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/x86/odp/api/abi/cpu_inlines.h \ arch/x86/odp/api/abi/cpu.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu.h b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu.h index 97a2861c5..825ff19d4 100644 --- a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu.h +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu.h @@ -17,15 +17,8 @@ extern "C" { #define ODP_CACHE_LINE_SIZE _ODP_CACHE_LINE_SIZE #endif -static inline void odp_cpu_pause(void) -{ - /* YIELD hints the CPU to switch to another thread if possible - * and executes as a NOP otherwise. - * ISB flushes the pipeline, then restarts. This is guaranteed to - * stall the CPU a number of cycles. - */ - __asm volatile("isb" ::: "memory"); -} +/* Inlined functions for non-ABI compat mode */ +#include #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_inlines.h new file mode 100644 index 000000000..bf44806a0 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_inlines.h @@ -0,0 +1,32 @@ +/* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_CPU_INLINES_H_ +#define ODP_ARCH_CPU_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_cpu_pause(void) +{ + /* YIELD hints the CPU to switch to another thread if possible + * and executes as a NOP otherwise. + * ISB flushes the pipeline, then restarts. This is guaranteed to + * stall the CPU a number of cycles. + */ + __asm volatile("isb" ::: "memory"); +} + +/* Use generic implementations for the rest of the functions */ +#include + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/arm/odp/api/abi/cpu.h b/platform/linux-generic/arch/arm/odp/api/abi/cpu.h index 0ab5b8e14..9224af9a0 100644 --- a/platform/linux-generic/arch/arm/odp/api/abi/cpu.h +++ b/platform/linux-generic/arch/arm/odp/api/abi/cpu.h @@ -13,15 +13,8 @@ extern "C" { #define ODP_CACHE_LINE_SIZE 64 -static inline void odp_cpu_pause(void) -{ - /* YIELD hints the CPU to switch to another thread if possible - * and executes as a NOP otherwise. - * ISB flushes the pipeline, then restarts. This is guaranteed to - * stall the CPU a number of cycles. - */ - __asm volatile("isb" ::: "memory"); -} +/* Inlined functions for non-ABI compat mode */ +#include #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/arm/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/arm/odp/api/abi/cpu_inlines.h new file mode 100644 index 000000000..bf44806a0 --- /dev/null +++ b/platform/linux-generic/arch/arm/odp/api/abi/cpu_inlines.h @@ -0,0 +1,32 @@ +/* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_CPU_INLINES_H_ +#define ODP_ARCH_CPU_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_cpu_pause(void) +{ + /* YIELD hints the CPU to switch to another thread if possible + * and executes as a NOP otherwise. + * ISB flushes the pipeline, then restarts. This is guaranteed to + * stall the CPU a number of cycles. + */ + __asm volatile("isb" ::: "memory"); +} + +/* Use generic implementations for the rest of the functions */ +#include + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu.h b/platform/linux-generic/arch/default/odp/api/abi/cpu.h index 8f64790c3..e09efdfcf 100644 --- a/platform/linux-generic/arch/default/odp/api/abi/cpu.h +++ b/platform/linux-generic/arch/default/odp/api/abi/cpu.h @@ -13,9 +13,8 @@ extern "C" { #define ODP_CACHE_LINE_SIZE 64 -static inline void odp_cpu_pause(void) -{ -} +/* Inlined functions for non-ABI compat mode */ +#include #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu_generic.h b/platform/linux-generic/arch/default/odp/api/abi/cpu_generic.h new file mode 100644 index 000000000..b75e65717 --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/cpu_generic.h @@ -0,0 +1,34 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_CPU_GENERIC_H_ +#define ODP_API_ABI_CPU_GENERIC_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +uint64_t _odp_cpu_cycles(void); +int _odp_cpu_cycles_init_global(void); + +static inline uint64_t _odp_cpu_cycles_max(void) +{ + return UINT64_MAX; +} + +static inline uint64_t _odp_cpu_cycles_resolution(void) +{ + return 1; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/cpu_inlines.h index f9c2bae9d..54aeae946 100644 --- a/platform/linux-generic/arch/default/odp/api/abi/cpu_inlines.h +++ b/platform/linux-generic/arch/default/odp/api/abi/cpu_inlines.h @@ -7,9 +7,18 @@ #ifndef ODP_ARCH_CPU_INLINES_H_ #define ODP_ARCH_CPU_INLINES_H_ -#undef odp_cpu_pause -#undef odp_cpu_cycles -#undef odp_cpu_cycles_max -#undef odp_cpu_cycles_resolution +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_cpu_pause(void) +{ +} + +#include + +#ifdef __cplusplus +} +#endif #endif diff --git a/platform/linux-generic/arch/default/odp_cpu_cycles.c b/platform/linux-generic/arch/default/odp_cpu_cycles.c index 542a68dbe..5d0d5db1d 100644 --- a/platform/linux-generic/arch/default/odp_cpu_cycles.c +++ b/platform/linux-generic/arch/default/odp_cpu_cycles.c @@ -1,4 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -6,16 +7,19 @@ #include +#include #include #include -#include #include +#include #include #define GIGA 1000000000 -uint64_t odp_cpu_cycles(void) +#include + +uint64_t _odp_cpu_cycles(void) { struct timespec time; uint64_t sec, ns, hz, cycles; @@ -26,7 +30,8 @@ uint64_t odp_cpu_cycles(void) if (ret != 0) ODP_ABORT("clock_gettime failed\n"); - hz = odp_cpu_hz_max(); + hz = odp_global_ro.system_info.cpu_hz_max[0]; + sec = (uint64_t)time.tv_sec; ns = (uint64_t)time.tv_nsec; @@ -36,16 +41,6 @@ uint64_t odp_cpu_cycles(void) return cycles; } -uint64_t odp_cpu_cycles_max(void) -{ - return UINT64_MAX; -} - -uint64_t odp_cpu_cycles_resolution(void) -{ - return 1; -} - int _odp_cpu_cycles_init_global(void) { return 0; diff --git a/platform/linux-generic/arch/mips64/odp/api/abi/cpu.h b/platform/linux-generic/arch/mips64/odp/api/abi/cpu.h index 45d95e9e3..a6ce523d0 100644 --- a/platform/linux-generic/arch/mips64/odp/api/abi/cpu.h +++ b/platform/linux-generic/arch/mips64/odp/api/abi/cpu.h @@ -17,13 +17,8 @@ extern "C" { #error Please add support for your arch in cpu_arch.h #endif -static inline void odp_cpu_pause(void) -{ - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); -} +/* Inlined functions for non-ABI compat mode */ +#include #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/mips64/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/mips64/odp/api/abi/cpu_inlines.h new file mode 100644 index 000000000..d3a424432 --- /dev/null +++ b/platform/linux-generic/arch/mips64/odp/api/abi/cpu_inlines.h @@ -0,0 +1,56 @@ +/* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_CPU_INLINES_H_ +#define ODP_ARCH_CPU_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +static inline void _odp_cpu_pause(void) +{ + __asm__ __volatile__ ("nop"); + __asm__ __volatile__ ("nop"); + __asm__ __volatile__ ("nop"); + __asm__ __volatile__ ("nop"); +} + +uint64_t _odp_cpu_cycles(void) +{ + #define CVMX_TMP_STR(x) CVMX_TMP_STR2(x) + #define CVMX_TMP_STR2(x) #x + uint64_t cycle; + + __asm__ __volatile__ ("rdhwr %[rt],$" CVMX_TMP_STR(31) : + [rt] "=d" (cycle) : : "memory"); + + return cycle; +} + +uint64_t _odp_cpu_cycles_max(void) +{ + return UINT64_MAX; +} + +uint64_t _odp_cpu_cycles_resolution(void) +{ + return 1; +} + +int _odp_cpu_cycles_init_global(void) +{ + return 0; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/mips64/odp_cpu_cycles.c b/platform/linux-generic/arch/mips64/odp_cpu_cycles.c deleted file mode 100644 index 6423af76d..000000000 --- a/platform/linux-generic/arch/mips64/odp_cpu_cycles.c +++ /dev/null @@ -1,37 +0,0 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include -#include -#include -#include - -uint64_t odp_cpu_cycles(void) -{ - #define CVMX_TMP_STR(x) CVMX_TMP_STR2(x) - #define CVMX_TMP_STR2(x) #x - uint64_t cycle; - - __asm__ __volatile__ ("rdhwr %[rt],$" CVMX_TMP_STR(31) : - [rt] "=d" (cycle) : : "memory"); - - return cycle; -} - -uint64_t odp_cpu_cycles_max(void) -{ - return UINT64_MAX; -} - -uint64_t odp_cpu_cycles_resolution(void) -{ - return 1; -} - -int _odp_cpu_cycles_init_global(void) -{ - return 0; -} diff --git a/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h b/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h index 9e3338d60..ecf56e82e 100644 --- a/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h +++ b/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h @@ -1,9 +1,25 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#define _ODP_NEED_GENERIC_CPU_PAUSE +#ifndef ODP_API_ABI_CPU_H_ +#define ODP_API_ABI_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + #define ODP_CACHE_LINE_SIZE 128 -#include + +/* Inlined functions for non-ABI compat mode */ +#include + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/platform/linux-generic/arch/x86/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/x86/odp/api/abi/cpu_inlines.h index 1eb8a9561..4b542a577 100644 --- a/platform/linux-generic/arch/x86/odp/api/abi/cpu_inlines.h +++ b/platform/linux-generic/arch/x86/odp/api/abi/cpu_inlines.h @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,10 +8,14 @@ #ifndef ODP_ARCH_CPU_INLINES_H_ #define ODP_ARCH_CPU_INLINES_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include #include -_ODP_INLINE void odp_cpu_pause(void) +static inline void _odp_cpu_pause(void) { #ifdef __SSE2__ __asm__ __volatile__ ("pause"); @@ -19,19 +24,23 @@ _ODP_INLINE void odp_cpu_pause(void) #endif } -_ODP_INLINE uint64_t odp_cpu_cycles(void) +static inline uint64_t _odp_cpu_cycles(void) { return _odp_cpu_rdtsc(); } -_ODP_INLINE uint64_t odp_cpu_cycles_max(void) +static inline uint64_t _odp_cpu_cycles_max(void) { return UINT64_MAX; } -_ODP_INLINE uint64_t odp_cpu_cycles_resolution(void) +static inline uint64_t _odp_cpu_cycles_resolution(void) { return 1; } +#ifdef __cplusplus +} +#endif + #endif diff --git a/platform/linux-generic/include/odp/api/plat/cpu_inlines.h b/platform/linux-generic/include/odp/api/plat/cpu_inlines.h index 054801e80..60c4bb920 100644 --- a/platform/linux-generic/include/odp/api/plat/cpu_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/cpu_inlines.h @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -9,6 +10,10 @@ #include +#include + +#include + /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ #ifndef _ODP_NO_INLINE @@ -23,7 +28,25 @@ #define _ODP_INLINE #endif -#include +_ODP_INLINE void odp_cpu_pause(void) +{ + _odp_cpu_pause(); +} + +_ODP_INLINE uint64_t odp_cpu_cycles_max(void) +{ + return _odp_cpu_cycles_max(); +} + +_ODP_INLINE uint64_t odp_cpu_cycles_resolution(void) +{ + return _odp_cpu_cycles_resolution(); +} + +_ODP_INLINE uint64_t odp_cpu_cycles(void) +{ + return _odp_cpu_cycles(); +} _ODP_INLINE uint64_t odp_cpu_cycles_diff(uint64_t c2, uint64_t c1) { -- cgit v1.2.3 From ce3dfec96e636cff3a84ecd11e166f3f85c7f324 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 6 Oct 2021 15:25:02 +0300 Subject: linux-gen: dpdk: packet parsing fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merge fixes done to the generic packet parsing code (ad9638fd9, 50fe59300, adae181a7) into the DPDK specific packet parsing implementation. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- platform/linux-generic/pktio/dpdk_parse.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/pktio/dpdk_parse.c b/platform/linux-generic/pktio/dpdk_parse.c index f593f4b11..c1b75f126 100644 --- a/platform/linux-generic/pktio/dpdk_parse.c +++ b/platform/linux-generic/pktio/dpdk_parse.c @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -53,7 +54,7 @@ static inline uint16_t dpdk_parse_eth(packet_parser_t *prs, eth = (const _odp_ethhdr_t *)*parseptr; /* Detect jumbo frames */ - if (odp_unlikely(frame_len > _ODP_ETH_LEN_MAX)) + if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX)) input_flags.jumbo = 1; /* Handle Ethernet broadcast/multicast addresses */ @@ -124,6 +125,16 @@ static inline uint16_t dpdk_parse_eth(packet_parser_t *prs, *parseptr += sizeof(_odp_vlanhdr_t); } + /* + * The packet was too short for what we parsed. We just give up + * entirely without trying to parse what fits in the packet. + */ + if (odp_unlikely(*offset > frame_len)) { + input_flags.all = 0; + input_flags.l2 = 1; + ethtype = 0; + } + error: prs->input_flags.all |= input_flags.all; @@ -151,6 +162,7 @@ static inline uint8_t dpdk_parse_ipv4(packet_parser_t *prs, if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN || ver != 4 || + sizeof(*ipv4) > frame_len - *offset || (l3_len > frame_len - *offset))) { prs->flags.ip_err = 1; return 0; @@ -218,8 +230,9 @@ static inline uint8_t dpdk_parse_ipv6(packet_parser_t *prs, uint32_t l4_packet_type = mbuf_packet_type & RTE_PTYPE_L4_MASK; /* Basic sanity checks on IPv6 header */ - if ((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || - l3_len > frame_len - *offset) { + if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || + sizeof(*ipv6) > frame_len - *offset || + l3_len > frame_len - *offset)) { prs->flags.ip_err = 1; return 0; } -- cgit v1.2.3 From 7a6de32fe370556276de53c4ede5ba947aa969cc Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 8 Oct 2021 11:26:12 +0300 Subject: linux-dpdk: crypto: try to handle segmented packets better MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Try to linearize segmented packets by shifting packet data in the first segment if it fits there with some headroom. This is done if the crypto device does not support chained mbufs. This will not help with large packets but is sufficient for the short segmented packets that are used in the current API validation tests. If linearization cannot be done, the crypto operation will fail. This change assumes that odp_crypto_operation() does not promise not to shift packet data within the packet when performing in-place crypto. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen Reviewed-by: Matias Elo --- platform/linux-dpdk/odp_crypto.c | 89 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 3 deletions(-) (limited to 'platform') diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 8ed561534..bcc1babef 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -64,8 +64,11 @@ typedef struct crypto_session_entry_s { struct rte_cryptodev_sym_session *rte_session; struct rte_crypto_sym_xform cipher_xform; struct rte_crypto_sym_xform auth_xform; + struct { + unsigned int cdev_qpairs_shared:1; + unsigned int chained_bufs_ok:1; + } flags; uint16_t cdev_nb_qpairs; - odp_bool_t cdev_qpairs_shared; uint8_t cdev_id; uint8_t cipher_iv_data[MAX_IV_LENGTH]; uint8_t auth_iv_data[MAX_IV_LENGTH]; @@ -1260,6 +1263,34 @@ check_finish: return -1; } +static int chained_bufs_ok(const odp_crypto_session_param_t *param, + uint8_t cdev_id) +{ + struct rte_cryptodev_info dev_info; + int chained_bufs_ok; + + rte_cryptodev_info_get(cdev_id, &dev_info); + chained_bufs_ok = !!(dev_info.feature_flags & RTE_CRYPTODEV_FF_IN_PLACE_SGL); + + /* + * Some crypto devices do not support chained buffers with all + * algorithms despite advertizing SG support in feature flags. + */ + + if (dev_info.driver_name && + !strcmp(dev_info.driver_name, "crypto_aesni_gcm") && + param->auth_alg == ODP_AUTH_ALG_AES_GMAC) + chained_bufs_ok = 0; + + if (dev_info.driver_name && + !strcmp(dev_info.driver_name, "crypto_openssl") && + (param->cipher_alg == ODP_CIPHER_ALG_AES_GCM || + param->auth_alg == ODP_AUTH_ALG_AES_GMAC)) + chained_bufs_ok = 0; + + return chained_bufs_ok; +} + #if RTE_VERSION < RTE_VERSION_NUM(19, 8, 0, 0) static int crypto_init_key(uint8_t **data, uint16_t *length, odp_crypto_key_t *key, const char *type) @@ -1482,6 +1513,7 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, param->auth_alg == ODP_AUTH_ALG_NULL) { rte_session = NULL; cdev_id = ~0; + session->flags.chained_bufs_ok = 1; session->cdev_nb_qpairs = 0; goto out_null; } else if (param->cipher_alg == ODP_CIPHER_ALG_NULL) { @@ -1524,8 +1556,12 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, goto err; } + session->flags.chained_bufs_ok = chained_bufs_ok(param, cdev_id); session->cdev_nb_qpairs = global->enabled_crypto_dev_qpairs[cdev_id]; - session->cdev_qpairs_shared = global->enabled_crypto_dev_qpairs_shared[cdev_id]; + if (global->enabled_crypto_dev_qpairs_shared[cdev_id]) + session->flags.cdev_qpairs_shared = 1; + else + session->flags.cdev_qpairs_shared = 0; out_null: session->rte_session = rte_session; session->cdev_id = cdev_id; @@ -1840,6 +1876,50 @@ static void crypto_fill_sym_param(crypto_session_entry_t *session, op->sym->auth.data.length = param->auth_range.length; } +/* + * Attempt to change a multi segment packet to a single segment packet by + * reducing the headroom. Shift packet data toward the start of the first + * segment and trim the tail, hopefully getting rid of the tail segment. + * + * This fails if the packet data does not fit in the first segment with + * the new headroom. A temporary copy to a bigger buffer would be needed + * in that case. + * + * Do nothing for single segment packets. + * + * We assume that odp_crypto_operation() makes no promise to not shift + * packet data within the packet. If that is not the case, the shifting + * done here needs to be undone after the crypto operation. + * + */ +static int linearize_pkt(const crypto_session_entry_t *session, odp_packet_t pkt) +{ + const uint32_t new_headroom = CONFIG_PACKET_HEADROOM; + uint32_t headroom; + uint32_t len; + uint32_t shift; + int rc; + + if (odp_likely(odp_packet_num_segs(pkt) == 1)) + return 0; + if (session->flags.chained_bufs_ok) + return 0; + + headroom = odp_packet_headroom(pkt); + if (odp_unlikely(new_headroom >= headroom)) + return -1; + + len = odp_packet_len(pkt); + shift = headroom - new_headroom; + odp_packet_push_head(pkt, shift); + odp_packet_move_data(pkt, 0, shift, len); + /* We rely on our trunc implementation to not change the handle */ + rc = odp_packet_trunc_tail(&pkt, shift, NULL, NULL); + ODP_ASSERT(rc == 0); + + return odp_packet_num_segs(pkt) != 1; +} + static int odp_crypto_int(odp_packet_t pkt_in, odp_packet_t *pkt_out, @@ -1896,6 +1976,9 @@ int odp_crypto_int(odp_packet_t pkt_in, pkt_in = ODP_PACKET_INVALID; } + if (linearize_pkt(session, out_pkt)) + goto err; + rte_session = session->rte_session; /* NULL rte_session means that it is a NULL-NULL operation. * Just return new packet. */ @@ -1914,7 +1997,7 @@ int odp_crypto_int(odp_packet_t pkt_in, int retry_count = 0; int queue_pair; int rc; - odp_bool_t queue_pairs_shared = session->cdev_qpairs_shared; + odp_bool_t queue_pairs_shared = session->flags.cdev_qpairs_shared; if (odp_unlikely(queue_pairs_shared)) queue_pair = odp_thread_id() % session->cdev_nb_qpairs; -- cgit v1.2.3 From 3017538dc811852b74079c9906ffa117a0ac9ee7 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Sun, 10 Oct 2021 19:21:41 +0300 Subject: linux-dpdk: packet: fix segment length setting in extend_head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When odp_packet_extend_head() prepends new segments in a packet, it will set the data length of the old head segment to the full buffer length, which is incorrect if the data does not extend to the end of the segment. Fix the problem by incrementing the data length of the old head segment by the old headroom, which is the actual amount of data added in the segment. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen Reviewed-by: Matias Elo --- platform/linux-dpdk/odp_packet.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'platform') diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index f5af3ce72..d094d5996 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -365,8 +365,8 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, } /* Expand the original head segment*/ newhead->pkt_len += rte_pktmbuf_headroom(mb); + mb->data_len += rte_pktmbuf_headroom(mb); mb->data_off = 0; - mb->data_len = mb->buf_len; _copy_head_metadata(newhead, mb); mb = newhead; *pkt = (odp_packet_t)newhead; -- cgit v1.2.3 From 7052433baac1ce14154dee3387c6a797acc9bad8 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Sun, 10 Oct 2021 23:24:32 +0300 Subject: linux-dpdk: packet: fix trunc_tail leaving last segment empty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit odp_packet_trunc_tail() uses rte_pktmbuf_trim() if possible, but that will succeed also if the trimmed length equals the last segment length, leaving the last segment empty of data. ODP API does not appear to allow such zero length packet segments even though it is not explicitly said. Fix the problem by calling rte_pktmbuf_trim() only after checking that the truncation length is strictly smaller than the last segment length. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen Reviewed-by: Matias Elo --- platform/linux-dpdk/odp_packet.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'platform') diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index d094d5996..b4105e0c4 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -522,11 +522,18 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, void **tail_ptr, uint32_t *tailroom) { struct rte_mbuf *mb = pkt_to_mbuf(*pkt); + struct rte_mbuf *last_mb = rte_pktmbuf_lastseg(mb); if (odp_unlikely(len >= odp_packet_len(*pkt))) return -1; - if (rte_pktmbuf_trim(mb, len)) { + /* + * Trim only if the last segment does not become zero length. + */ + if (odp_likely(len < last_mb->data_len)) { + if (odp_unlikely(rte_pktmbuf_trim(mb, len))) + return -1; + } else { struct rte_mbuf *reverse[mb->nb_segs]; struct rte_mbuf *t = mb; int i; -- cgit v1.2.3 From 2a57ec26d5897a8d216d2329bb66d774288f2cfc Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 13 Sep 2021 11:46:07 +0300 Subject: Port cb8d8942b "linux-gen: sched: limit interface header visibility" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- platform/linux-dpdk/include/odp_buffer_internal.h | 1 - 1 file changed, 1 deletion(-) (limited to 'platform') diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h b/platform/linux-dpdk/include/odp_buffer_internal.h index 37e5d1b4f..e47030ea7 100644 --- a/platform/linux-dpdk/include/odp_buffer_internal.h +++ b/platform/linux-dpdk/include/odp_buffer_internal.h @@ -29,7 +29,6 @@ extern "C" { #include #include #include -#include #include /* DPDK */ -- cgit v1.2.3 From 225a3f9e8c478a57e855e80724ebeda33362a785 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 13 Sep 2021 12:01:38 +0300 Subject: Port c6f4b3a46 "linux-gen: sched: add scheduler identifier" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- platform/linux-dpdk/include/odp_eventdev_internal.h | 2 ++ platform/linux-dpdk/odp_schedule_if.c | 8 ++++++++ 2 files changed, 10 insertions(+) (limited to 'platform') diff --git a/platform/linux-dpdk/include/odp_eventdev_internal.h b/platform/linux-dpdk/include/odp_eventdev_internal.h index 496a2238f..9e1083fd5 100644 --- a/platform/linux-dpdk/include/odp_eventdev_internal.h +++ b/platform/linux-dpdk/include/odp_eventdev_internal.h @@ -36,6 +36,8 @@ extern "C" { #include +#define _ODP_SCHED_ID_EVENTDEV (_ODP_SCHED_ID_SCALABLE + 1) + #define RX_ADAPTER_INIT 0 #define RX_ADAPTER_STOPPED 1 #define RX_ADAPTER_RUNNING 2 diff --git a/platform/linux-dpdk/odp_schedule_if.c b/platform/linux-dpdk/odp_schedule_if.c index 29d38b1fd..21f5fc85b 100644 --- a/platform/linux-dpdk/odp_schedule_if.c +++ b/platform/linux-dpdk/odp_schedule_if.c @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -11,6 +12,9 @@ #include #include +/* Required for _ODP_SCHED_ID_EVENTDEV */ +#include + #include #include @@ -25,6 +29,7 @@ extern const schedule_api_t _odp_schedule_eventdev_api; const schedule_fn_t *_odp_sched_fn; const schedule_api_t *_odp_sched_api; +int _odp_sched_id; uint64_t odp_schedule_wait_time(uint64_t ns) { @@ -216,12 +221,15 @@ int _odp_schedule_init_global(void) ODP_PRINT("Using scheduler '%s'\n", sched); if (!strcmp(sched, "basic")) { + _odp_sched_id = _ODP_SCHED_ID_BASIC; _odp_sched_fn = &_odp_schedule_basic_fn; _odp_sched_api = &_odp_schedule_basic_api; } else if (!strcmp(sched, "sp")) { + _odp_sched_id = _ODP_SCHED_ID_SP; _odp_sched_fn = &_odp_schedule_sp_fn; _odp_sched_api = &_odp_schedule_sp_api; } else if (!strcmp(sched, "eventdev")) { + _odp_sched_id = _ODP_SCHED_ID_EVENTDEV; _odp_sched_fn = &_odp_schedule_eventdev_fn; _odp_sched_api = &_odp_schedule_eventdev_api; } else { -- cgit v1.2.3 From 6c439e2804771c9da2d76d5c9175f8784c2e0200 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 13 Sep 2021 12:09:01 +0300 Subject: Port 3c40dfc7a "linux-gen: queue: improve queue debug print" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- .../linux-dpdk/include/odp_queue_basic_internal.h | 3 ++ platform/linux-dpdk/odp_queue_basic.c | 32 ++++++++++++++++++---- 2 files changed, 29 insertions(+), 6 deletions(-) (limited to 'platform') diff --git a/platform/linux-dpdk/include/odp_queue_basic_internal.h b/platform/linux-dpdk/include/odp_queue_basic_internal.h index c3ddaf334..1e542d973 100644 --- a/platform/linux-dpdk/include/odp_queue_basic_internal.h +++ b/platform/linux-dpdk/include/odp_queue_basic_internal.h @@ -116,6 +116,9 @@ int _odp_sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int num, int update_status); int _odp_sched_queue_empty(uint32_t queue_index); +/* Functions by schedulers */ +int _odp_sched_basic_get_spread(uint32_t queue_index); + #ifdef __cplusplus } #endif diff --git a/platform/linux-dpdk/odp_queue_basic.c b/platform/linux-dpdk/odp_queue_basic.c index 7145fb2ce..c5e30d774 100644 --- a/platform/linux-dpdk/odp_queue_basic.c +++ b/platform/linux-dpdk/odp_queue_basic.c @@ -634,7 +634,8 @@ static void queue_print(odp_queue_t handle) odp_pktio_info_t pktio_info; queue_entry_t *queue; uint32_t queue_id; - int status; + int status, prio; + int max_prio = odp_schedule_max_prio(); queue_id = queue_to_index(handle); @@ -658,7 +659,7 @@ static void queue_print(odp_queue_t handle) ODP_PRINT("\nQueue info\n"); ODP_PRINT("----------\n"); ODP_PRINT(" handle %p\n", queue->s.handle); - ODP_PRINT(" index %" PRIu32 "\n", queue->s.index); + ODP_PRINT(" index %" PRIu32 "\n", queue_id); ODP_PRINT(" name %s\n", queue->s.name); ODP_PRINT(" enq mode %s\n", queue->s.param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : @@ -686,8 +687,11 @@ static void queue_print(odp_queue_t handle) "ODP_SCHED_SYNC_ATOMIC" : (queue->s.param.sched.sync == ODP_SCHED_SYNC_ORDERED ? "ODP_SCHED_SYNC_ORDERED" : "unknown"))); - ODP_PRINT(" priority %d\n", queue->s.param.sched.prio); - ODP_PRINT(" group %d\n", queue->s.param.sched.group); + prio = queue->s.param.sched.prio; + ODP_PRINT(" priority %i (%i in API)\n", max_prio - prio, prio); + ODP_PRINT(" group %i\n", queue->s.param.sched.group); + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + ODP_PRINT(" spread %i\n", _odp_sched_basic_get_spread(queue_id)); } if (queue->s.pktin.pktio != ODP_PKTIO_INVALID) { if (!odp_pktio_info(queue->s.pktin.pktio, &pktio_info)) @@ -744,11 +748,18 @@ static void queue_print_all(void) char type_c, enq_c, deq_c, order_c, sync_c; const int col_width = 24; int prio = 0; + int spr = 0; odp_schedule_sync_t sync = ODP_SCHED_SYNC_PARALLEL; + odp_schedule_group_t grp = ODP_SCHED_GROUP_INVALID; ODP_PRINT("\nList of all queues\n"); ODP_PRINT("------------------\n"); - ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio\n", col_width, "name"); + ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio grp", + col_width, "name"); + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + ODP_PRINT(" spr\n"); + else + ODP_PRINT("\n"); for (i = 0; i < CONFIG_MAX_QUEUES; i++) { queue_entry_t *queue = qentry_from_index(i); @@ -777,7 +788,10 @@ static void queue_print_all(void) len = ring_st_length(queue->s.ring_st); max_len = ring_st_max_length(queue->s.ring_st); prio = queue->s.param.sched.prio; + grp = queue->s.param.sched.group; sync = queue->s.param.sched.sync; + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + spr = _odp_sched_basic_get_spread(index); } else { len = ring_mpmc_length(queue->s.ring_mpmc); max_len = ring_mpmc_max_length(queue->s.ring_mpmc); @@ -811,7 +825,13 @@ static void queue_print_all(void) if (type == ODP_QUEUE_TYPE_SCHED) { sync_c = (sync == ODP_SCHED_SYNC_PARALLEL) ? 'P' : ((sync == ODP_SCHED_SYNC_ATOMIC) ? 'A' : 'O'); - ODP_PRINT(" %c %4i", sync_c, prio); + /* Print prio level matching odp_schedule_print() output */ + prio = odp_schedule_max_prio() - prio; + + ODP_PRINT(" %c %4i %3i", sync_c, prio, grp); + + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + ODP_PRINT(" %3i", spr); } ODP_PRINT("\n"); -- cgit v1.2.3 From 9eebac9f3d0d8761ddf7e062920b7cf326990339 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 5 Oct 2021 15:21:26 +0300 Subject: Port bf4f2390d "linux-gen: packet: implement minimal odp_packet_reass_info()" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- platform/linux-dpdk/odp_packet.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'platform') diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index b4105e0c4..67f91821d 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -2411,6 +2411,13 @@ odp_packet_reass_status_t odp_packet_reass_status(odp_packet_t pkt) return ODP_PACKET_REASS_NONE; } +int odp_packet_reass_info(odp_packet_t pkt, odp_packet_reass_info_t *info) +{ + (void)pkt; + (void)info; + return -1; +} + int odp_packet_reass_partial_state(odp_packet_t pkt, odp_packet_t frags[], odp_packet_reass_partial_state_t *res) { -- cgit v1.2.3 From 5b0c6b578a8d3b5f58116fbe5bf26757524af8a2 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 6 Oct 2021 11:15:33 +0300 Subject: Port f36c9ec60 "linux-gen: shm: rename implementation internal shm blocks" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- platform/linux-dpdk/odp_crypto.c | 2 +- platform/linux-dpdk/odp_pool.c | 2 +- platform/linux-dpdk/odp_queue_basic.c | 2 +- platform/linux-dpdk/odp_queue_eventdev.c | 2 +- platform/linux-dpdk/odp_thread.c | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) (limited to 'platform') diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index bcc1babef..13e646230 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -315,7 +315,7 @@ int _odp_crypto_init_global(void) mem_size += (MAX_SESSIONS * sizeof(crypto_session_entry_t)); /* Allocate our globally shared memory */ - shm = odp_shm_reserve("_odp_crypto_glb", mem_size, + shm = odp_shm_reserve("_odp_crypto_global", mem_size, ODP_CACHE_LINE_SIZE, 0); if (shm != ODP_SHM_INVALID) { global = odp_shm_addr(shm); diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index 4b5eb3206..4417ca4ee 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -153,7 +153,7 @@ int _odp_pool_init_global(void) uint32_t i; odp_shm_t shm; - shm = odp_shm_reserve("_odp_pool_glb", sizeof(pool_global_t), + shm = odp_shm_reserve("_odp_pool_global", sizeof(pool_global_t), ODP_CACHE_LINE_SIZE, 0); _odp_pool_glb = odp_shm_addr(shm); diff --git a/platform/linux-dpdk/odp_queue_basic.c b/platform/linux-dpdk/odp_queue_basic.c index c5e30d774..33e0ba6bc 100644 --- a/platform/linux-dpdk/odp_queue_basic.c +++ b/platform/linux-dpdk/odp_queue_basic.c @@ -130,7 +130,7 @@ static int queue_init_global(void) _odp_queue_inline_offset.context = offsetof(queue_entry_t, s.param.context); - shm = odp_shm_reserve("_odp_queue_gbl", + shm = odp_shm_reserve("_odp_queue_basic_global", sizeof(queue_global_t), sizeof(queue_entry_t), 0); diff --git a/platform/linux-dpdk/odp_queue_eventdev.c b/platform/linux-dpdk/odp_queue_eventdev.c index 96baffa6f..0960c456e 100644 --- a/platform/linux-dpdk/odp_queue_eventdev.c +++ b/platform/linux-dpdk/odp_queue_eventdev.c @@ -528,7 +528,7 @@ static int queue_init_global(void) _odp_queue_inline_offset.context = offsetof(queue_entry_t, s.param.context); - shm = odp_shm_reserve("_odp_eventdev_gbl", + shm = odp_shm_reserve("_odp_queue_eventdev_global", sizeof(eventdev_global_t), ODP_CACHE_LINE_SIZE, 0); diff --git a/platform/linux-dpdk/odp_thread.c b/platform/linux-dpdk/odp_thread.c index 59394b3e4..7ab41cf72 100644 --- a/platform/linux-dpdk/odp_thread.c +++ b/platform/linux-dpdk/odp_thread.c @@ -76,7 +76,7 @@ int _odp_thread_init_global(void) if (num_max > ODP_THREAD_COUNT_MAX) num_max = ODP_THREAD_COUNT_MAX; - shm = odp_shm_reserve("_odp_thread_globals", + shm = odp_shm_reserve("_odp_thread_global", sizeof(thread_globals_t), ODP_CACHE_LINE_SIZE, 0); -- cgit v1.2.3 From c5a3ebc20362f7c7d21598304ccc9de24e2e84f4 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 6 Oct 2021 11:38:52 +0300 Subject: Port c6339d406 "linux-gen: shm: allocate enough resources for internal allocations" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- platform/linux-dpdk/include/odp_config_internal.h | 13 ++++++++++--- platform/linux-dpdk/odp_shared_memory.c | 21 ++++++++++----------- platform/linux-dpdk/odp_system_info.c | 2 +- 3 files changed, 21 insertions(+), 15 deletions(-) (limited to 'platform') diff --git a/platform/linux-dpdk/include/odp_config_internal.h b/platform/linux-dpdk/include/odp_config_internal.h index 73d2304c9..6618d413d 100644 --- a/platform/linux-dpdk/include/odp_config_internal.h +++ b/platform/linux-dpdk/include/odp_config_internal.h @@ -124,11 +124,18 @@ extern "C" { CONFIG_PACKET_HEADROOM - \ CONFIG_PACKET_TAILROOM) -/* Maximum number of shared memory blocks. +/* + * Number of shared memory blocks reserved for implementation internal use. + */ +#define CONFIG_INTERNAL_SHM_BLOCKS 20 + +/* + * Maximum number of shared memory blocks. * - * This the number of separate SHM areas that can be reserved concurrently + * This is the number of separate SHM blocks that an application can reserve + * concurrently. */ -#define ODP_CONFIG_SHM_BLOCKS (ODP_CONFIG_POOLS + 48) +#define CONFIG_SHM_BLOCKS 64 /* * Maximum event burst size diff --git a/platform/linux-dpdk/odp_shared_memory.c b/platform/linux-dpdk/odp_shared_memory.c index 4b1432a2f..645bb8847 100644 --- a/platform/linux-dpdk/odp_shared_memory.c +++ b/platform/linux-dpdk/odp_shared_memory.c @@ -1,4 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -25,9 +26,7 @@ #define SHM_MAX_ALIGN (0x80000000) #define SHM_BLOCK_NAME "%" PRIu64 "-%d-%s" - -ODP_STATIC_ASSERT(ODP_CONFIG_SHM_BLOCKS >= ODP_CONFIG_POOLS, - "ODP_CONFIG_SHM_BLOCKS < ODP_CONFIG_POOLS"); +#define SHM_MAX_NB_BLOCKS (CONFIG_INTERNAL_SHM_BLOCKS + CONFIG_SHM_BLOCKS) ODP_STATIC_ASSERT(ODP_SHM_NAME_LEN >= RTE_MEMZONE_NAMESIZE, "ODP_SHM_NAME_LEN < RTE_MEMZONE_NAMESIZE"); @@ -68,7 +67,7 @@ typedef struct { */ typedef struct { odp_spinlock_t lock; - shm_block_t block[ODP_CONFIG_SHM_BLOCKS]; + shm_block_t block[SHM_MAX_NB_BLOCKS]; } shm_table_t; static shm_table_t *shm_tbl; @@ -80,7 +79,7 @@ static odp_bool_t mz_name_used(const char *name) { int idx; - for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) { + for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) { if (shm_tbl->block[idx].mz && strncmp(name, shm_tbl->block[idx].mz->name, RTE_MEMZONE_NAMESIZE) == 0) @@ -124,7 +123,7 @@ static int find_free_block(void) { int idx; - for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) { + for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) { if (shm_tbl->block[idx].mz == NULL) return idx; } @@ -145,7 +144,7 @@ static inline odp_bool_t handle_is_valid(odp_shm_t shm) { int idx = handle_to_idx(shm); - if (idx < 0 || idx >= ODP_CONFIG_SHM_BLOCKS || + if (idx < 0 || idx >= SHM_MAX_NB_BLOCKS || shm_tbl->block[idx].mz == NULL) { ODP_ERR("Invalid odp_shm_t handle: %" PRIu64 "\n", odp_shm_to_u64(shm)); @@ -199,7 +198,7 @@ int _odp_shm_term_global(void) } /* Cleanup possibly non freed memory (and complain a bit) */ - for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) { + for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) { block = &shm_tbl->block[idx]; if (block->mz) { ODP_ERR("block '%s' was never freed (cleaning up...)\n", @@ -224,7 +223,7 @@ int odp_shm_capability(odp_shm_capability_t *capa) { memset(capa, 0, sizeof(odp_shm_capability_t)); - capa->max_blocks = ODP_CONFIG_SHM_BLOCKS; + capa->max_blocks = CONFIG_SHM_BLOCKS; capa->max_size = 0; capa->max_align = SHM_MAX_ALIGN; @@ -357,7 +356,7 @@ odp_shm_t odp_shm_lookup(const char *name) odp_spinlock_lock(&shm_tbl->lock); - for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) { + for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) { if (shm_tbl->block[idx].mz && strncmp(name, shm_tbl->block[idx].name, ODP_SHM_NAME_LEN) == 0) { @@ -425,7 +424,7 @@ void odp_shm_print_all(void) ODP_PRINT("\nShared memory blocks\n--------------------\n"); - for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) { + for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) { block = &shm_tbl->block[idx]; if (block->mz == NULL) continue; diff --git a/platform/linux-dpdk/odp_system_info.c b/platform/linux-dpdk/odp_system_info.c index af07f387a..30fcc140b 100644 --- a/platform/linux-dpdk/odp_system_info.c +++ b/platform/linux-dpdk/odp_system_info.c @@ -515,7 +515,7 @@ void odp_sys_config_print(void) ODP_PRINT("CONFIG_PACKET_SEG_SIZE: %i\n", CONFIG_PACKET_SEG_SIZE); ODP_PRINT("CONFIG_PACKET_SEG_LEN_MIN: %i\n", CONFIG_PACKET_SEG_LEN_MIN); ODP_PRINT("CONFIG_PACKET_MAX_SEG_LEN: %i\n", CONFIG_PACKET_MAX_SEG_LEN); - ODP_PRINT("ODP_CONFIG_SHM_BLOCKS: %i\n", ODP_CONFIG_SHM_BLOCKS); + ODP_PRINT("CONFIG_SHM_BLOCKS: %i\n", CONFIG_SHM_BLOCKS); ODP_PRINT("CONFIG_BURST_SIZE: %i\n", CONFIG_BURST_SIZE); ODP_PRINT("CONFIG_POOL_MAX_NUM: %i\n", CONFIG_POOL_MAX_NUM); ODP_PRINT("\n"); -- cgit v1.2.3 From c7be5c1894d38158496a5ec0dbe99ca08bcb02cc Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 6 Oct 2021 11:45:20 +0300 Subject: Port 2bc648570 "linux-gen: stash: add support for disabling stash feature" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- platform/linux-dpdk/odp_init.c | 1 + 1 file changed, 1 insertion(+) (limited to 'platform') diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c index 914a48fad..45f00f8ac 100644 --- a/platform/linux-dpdk/odp_init.c +++ b/platform/linux-dpdk/odp_init.c @@ -74,6 +74,7 @@ static void disable_features(odp_global_data_ro_t *global_ro, if (disable_ipsec && disable_crypto) global_ro->disable.crypto = 1; + global_ro->disable.stash = init_param->not_used.feat.stash; global_ro->disable.traffic_mngr = init_param->not_used.feat.tm; global_ro->disable.compress = init_param->not_used.feat.compress; } -- cgit v1.2.3 From 100568dfb9e06227ccbf995e45ac343b44f61405 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 6 Oct 2021 12:51:37 +0300 Subject: Port ad9638fd9 "linux-gen: packet: fix odp_packet_parse() with non-zero offset" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- platform/linux-dpdk/odp_packet.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'platform') diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index 67f91821d..027f8027c 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -1335,7 +1335,7 @@ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, eth = (const _odp_ethhdr_t *)*parseptr; /* Detect jumbo frames */ - if (odp_unlikely(frame_len > _ODP_ETH_LEN_MAX)) + if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX)) input_flags.jumbo = 1; /* Handle Ethernet broadcast/multicast addresses */ -- cgit v1.2.3 From 031e525ac229e80b68fc626575ef09215ec37c38 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 6 Oct 2021 12:55:08 +0300 Subject: Port 50fe59300 "linux-gen: packet: fix Ethernet frame length checking in parsing" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- platform/linux-dpdk/odp_packet.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'platform') diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index 027f8027c..9774a25bf 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -1393,6 +1393,16 @@ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_vlanhdr_t); } + /* + * The packet was too short for what we parsed. We just give up + * entirely without trying to parse what fits in the packet. + */ + if (odp_unlikely(*offset > frame_len)) { + input_flags.all = 0; + input_flags.l2 = 1; + ethtype = 0; + } + error: prs->input_flags.all |= input_flags.all; -- cgit v1.2.3 From b406eee3254b33df2260ef7726c4ad97d64306d9 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 6 Oct 2021 12:59:02 +0300 Subject: Port adae181a7 "linux-gen: packet: fix packet length checking in IP parsing" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- platform/linux-dpdk/odp_packet.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'platform') diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index 9774a25bf..62b80bf2e 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -1426,6 +1426,7 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN || ver != 4 || + sizeof(*ipv4) > frame_len - *offset || (l3_len > frame_len - *offset))) { prs->flags.ip_err = 1; return 0; @@ -1484,8 +1485,9 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, _ODP_IPV6HDR_LEN; /* Basic sanity checks on IPv6 header */ - if ((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || - l3_len > frame_len - *offset) { + if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || + sizeof(*ipv6) > frame_len - *offset || + l3_len > frame_len - *offset)) { prs->flags.ip_err = 1; return 0; } -- cgit v1.2.3 From ad0994dc817c6a6aebe533ee620ab171c7a0b114 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 6 Oct 2021 13:08:52 +0300 Subject: =?UTF-8?q?Port=20b70771a1d=20=E2=80=9Clinux-gen:=20packet:=20fix?= =?UTF-8?q?=20SCTP=20checksum=20check=20with=20multi-segment=20packets"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- platform/linux-dpdk/odp_packet.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'platform') diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index 62b80bf2e..c7e9061d3 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -2003,6 +2003,8 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, if (chksums.chksum.sctp && pkt_hdr->p.input_flags.sctp && !pkt_hdr->p.input_flags.ipfrag) { + uint32_t seg_len = 0; + _odp_sctphdr_t hdr_copy; uint32_t sum = ~packet_sum_crc32c(pkt_hdr, pkt_hdr->p.l4_offset + _ODP_SCTPHDR_LEN, @@ -2012,8 +2014,14 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, l4_part_sum); _odp_sctphdr_t *sctp = odp_packet_offset(packet_handle(pkt_hdr), pkt_hdr->p.l4_offset, - NULL, NULL); + &seg_len, NULL); + if (odp_unlikely(seg_len < sizeof(*sctp))) { + odp_packet_t pkt = packet_handle(pkt_hdr); + sctp = &hdr_copy; + odp_packet_copy_to_mem(pkt, pkt_hdr->p.l4_offset, + sizeof(*sctp), sctp); + } pkt_hdr->p.input_flags.l4_chksum_done = 1; if (sum != sctp->chksum) { pkt_hdr->p.flags.l4_chksum_err = 1; -- cgit v1.2.3 From b24ae39d270ba998760430079caf5772374602ba Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 6 Oct 2021 13:26:12 +0300 Subject: Port 6b93469a7 "linux-gen: packet: fix parsing of segmented packets in odp_packet_parse()" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- platform/linux-dpdk/odp_packet.c | 60 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 4 deletions(-) (limited to 'platform') diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index c7e9061d3..065a182a8 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -1318,7 +1319,15 @@ static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr, return sum; } -/** Parser helper function for Ethernet packets */ +/* + * In the worst case we look at the Ethernet header, 8 bytes of LLC/SNAP + * header and two VLAN tags in the same packet. + */ +#define PARSE_ETH_BYTES (sizeof(_odp_ethhdr_t) + 8 + 2 * sizeof(_odp_vlanhdr_t)) +/** Parser helper function for Ethernet packets + * + * Requires up to PARSE_ETH_BYTES bytes of contiguous packet data. + */ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len) { @@ -1409,8 +1418,11 @@ error: return ethtype; } +#define PARSE_IPV4_BYTES (0xfU * 4) /* max IPv4 header length with options */ /** * Parser helper function for IPv4 + * + * Requires up to PARSE_IPV4_BYTES bytes of contiguous packet data. */ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, @@ -1469,8 +1481,15 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, return ipv4->proto; } +/* + * Peeks 2 bytes beyond IPv6 base header without length check if there + * are extension headers. + */ +#define PARSE_IPV6_BYTES (sizeof(_odp_ipv6hdr_t) + 2) /** * Parser helper function for IPv6 + * + * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data. */ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, @@ -1539,8 +1558,11 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, return ipv6->next_hdr; } +#define PARSE_TCP_BYTES (sizeof(_odp_tcphdr_t)) /** * Parser helper function for TCP + * + * Requires PARSE_TCP_BYTES bytes of contiguous packet data. */ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t tcp_len, @@ -1566,8 +1588,15 @@ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += len; } +/* + * In the worst case we look at the UDP header and 4 bytes of the UDP + * payload (the non-ESP marker to distinguish IKE packets from ESP packets). + */ +#define PARSE_UDP_BYTES (sizeof(_odp_udphdr_t) + 4) /** * Parser helper function for UDP + * + * Requires PARSE_UDP_BYTES bytes of contiguous packet data. */ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, odp_proto_chksums_t chksums, @@ -1612,8 +1641,11 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_udphdr_t); } +#define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t)) /** * Parser helper function for SCTP + * + * Requires PARSE_SCTP_BYTES bytes of contiguous packet data. */ static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t sctp_len, @@ -1640,6 +1672,10 @@ static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_sctphdr_t); } +#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c))) +#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \ + MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES)) +/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */ static inline int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, uint32_t offset, @@ -2078,12 +2114,14 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); const uint8_t *data; uint32_t seg_len; - uint32_t len = odp_packet_len(pkt); + uint32_t packet_len = odp_packet_len(pkt); odp_proto_t proto = param->proto; odp_proto_layer_t layer = param->last_layer; int ret; uint16_t ethtype; uint64_t l4_part_sum = 0; + const uint32_t min_seglen = PARSE_ETH_BYTES + PARSE_L3_L4_BYTES; + uint8_t buf[min_seglen]; if (proto == ODP_PROTO_NONE || layer == ODP_PROTO_LAYER_NONE) return -1; @@ -2093,6 +2131,20 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, if (data == NULL) return -1; + /* + * We must not have a packet segment boundary within the parsed + * packet data range. Copy enough data to a temporary buffer for + * parsing if necessary. + */ + if (odp_unlikely(pkt_hdr->buf_hdr.mb.nb_segs > 1) && + odp_unlikely(seg_len < min_seglen)) { + seg_len = min_seglen; + if (seg_len > packet_len - offset) + seg_len = packet_len - offset; + odp_packet_copy_to_mem(pkt, offset, seg_len, buf); + data = buf; + } + /* Reset parser flags, keep other flags */ packet_parse_reset(pkt_hdr, 0); @@ -2100,7 +2152,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, /* Assume valid L2 header, no CRC/FCS check in SW */ pkt_hdr->p.l2_offset = offset; - ethtype = parse_eth(&pkt_hdr->p, &data, &offset, len); + ethtype = parse_eth(&pkt_hdr->p, &data, &offset, packet_len); } else if (proto == ODP_PROTO_IPV4) { ethtype = _ODP_ETHTYPE_IPV4; } else if (proto == ODP_PROTO_IPV6) { @@ -2110,7 +2162,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, } ret = packet_parse_common_l3_l4(&pkt_hdr->p, data, offset, - len, seg_len, + packet_len, seg_len, layer, ethtype, param->chksums, &l4_part_sum); -- cgit v1.2.3 From ec77bf59d3407278625c28a8d47c3dab9627a91f Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 6 Oct 2021 14:22:53 +0300 Subject: Port a369bcd19 "linux-gen: cpu: refactor inlining code" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- platform/linux-dpdk/Makefile.am | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) (limited to 'platform') diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 61b61f6d6..279aed0c1 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -242,12 +242,13 @@ __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ arch/default/odp_hash_crc32.c \ arch/arm/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/arm/odp/api/abi/cpu_inlines.h \ arch/arm/odp/api/abi/cpu.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ @@ -264,14 +265,15 @@ __LIB__libodp_dpdk_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/aarch64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/aarch64/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu_time.h \ arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/aarch64/odp/api/abi/atomic_inlines.h \ arch/aarch64/odp/api/abi/atomic.h \ - arch/aarch64/odp/api/abi/cpu.h + arch/default/odp/api/abi/cpu_generic.h \ + arch/aarch64/odp/api/abi/cpu_inlines.h \ + arch/aarch64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ @@ -284,12 +286,13 @@ __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -298,16 +301,16 @@ noinst_HEADERS += arch/default/odp_atomic.h \ endif if ARCH_IS_MIPS64 __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ - arch/mips64/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/mips64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/mips64/odp/api/abi/cpu_inlines.h \ arch/mips64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -320,12 +323,13 @@ __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/powerpc/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/default/odp/api/abi/cpu_inlines.h \ arch/powerpc/odp/api/abi/cpu.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -339,13 +343,13 @@ __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/x86/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_inlines.h \ - arch/x86/odp/api/abi/cpu_rdtsc.h \ +odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_rdtsc.h \ arch/x86/odp/api/abi/cpu_time.h \ arch/x86/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/x86/odp/api/abi/cpu_inlines.h \ arch/x86/odp/api/abi/cpu.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ -- cgit v1.2.3