diff options
author | Matias Elo <matias.elo@nokia.com> | 2022-05-04 15:40:33 +0300 |
---|---|---|
committer | Matias Elo <matias.elo@nokia.com> | 2022-05-04 15:40:33 +0300 |
commit | 2f0df4cb4acdee52f436cad8476ccf662821b7b5 (patch) | |
tree | 085136f3d98d8187b17ea584c992c8117bdedd8f /platform | |
parent | 2f869d2698349bf75ec2404d4ab7d7f3b5b4c88b (diff) | |
parent | e361227ac00851e8720871e927b015c4c0e6a895 (diff) |
Merge tag 'v1.36.0.0' of https://github.com/OpenDataPlane/odp into odp-dpdk
Signed-off-by: Matias Elo <matias.elo@nokia.com>
Diffstat (limited to 'platform')
58 files changed, 1981 insertions, 1571 deletions
diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 47e5cf759..d4e94f677 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -49,7 +49,9 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/sync_inlines.h \ include/odp/api/plat/thread_inlines.h \ include/odp/api/plat/ticketlock_inlines.h \ - include/odp/api/plat/time_inlines.h + include/odp/api/plat/time_inlines.h \ + include/odp/api/plat/timer_inlines.h \ + include/odp/api/plat/timer_inline_types.h odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/align.h \ @@ -65,6 +67,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/dma_types.h \ include-abi/odp/api/abi/errno.h \ include-abi/odp/api/abi/event.h \ + include-abi/odp/api/abi/event_types.h \ include-abi/odp/api/abi/hash.h \ include-abi/odp/api/abi/init.h \ include-abi/odp/api/abi/ipsec.h \ @@ -95,6 +98,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/thrmask.h \ include-abi/odp/api/abi/ticketlock.h \ include-abi/odp/api/abi/time.h \ + include-abi/odp/api/abi/timer.h \ include-abi/odp/api/abi/timer_types.h \ include-abi/odp/api/abi/traffic_mngr.h \ include-abi/odp/api/abi/version.h @@ -126,6 +130,7 @@ noinst_HEADERS = \ include/odp_packet_dpdk.h \ include/odp_packet_internal.h \ include/odp_packet_io_internal.h \ + include/odp_parse_internal.h \ include/odp_socket_common.h \ include/odp_packet_io_stats_common.h \ include/odp_packet_io_stats.h \ @@ -160,6 +165,7 @@ noinst_HEADERS = \ include/odp_timer_internal.h \ include/odp_timer_wheel_internal.h \ include/odp_traffic_mngr_internal.h \ + include/odp_types_internal.h \ include/odp_event_vector_internal.h \ include/protocols/eth.h \ include/protocols/ip.h \ @@ -201,6 +207,7 @@ __LIB__libodp_linux_la_SOURCES = \ odp_packet_vector.c \ odp_packet_flags.c \ odp_packet_io.c \ + odp_parse.c \ odp_pkt_queue.c \ odp_pool.c \ odp_queue_basic.c \ @@ -279,7 +286,8 @@ __LIB__libodp_linux_la_SOURCES += \ odp_sync_api.c \ odp_thread_api.c \ odp_ticketlock_api.c \ - odp_time_api.c + odp_time_api.c \ + odp_timer_api.c endif if ARCH_IS_ARM diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.h b/platform/linux-generic/arch/aarch64/odp_atomic.h index 445d91e28..d3b8ea4dc 100644 --- a/platform/linux-generic/arch/aarch64/odp_atomic.h +++ b/platform/linux-generic/arch/aarch64/odp_atomic.h @@ -12,6 +12,7 @@ #error This file should not be included directly, please include odp_cpu.h #endif +#include <odp_types_internal.h> #include <limits.h> #ifdef CONFIG_DMBSTR @@ -36,22 +37,18 @@ do { \ #define LL_MO(mo) (HAS_ACQ((mo)) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define SC_MO(mo) (HAS_RLS((mo)) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED) -/* Prevent warnings about ISO C not supporting __int128 */ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" - #ifndef __ARM_FEATURE_QRDMX /* Feature only available in v8.1a and beyond */ static inline bool -__lockfree_compare_exchange_16(register __int128 *var, __int128 *exp, - register __int128 neu, bool weak, int mo_success, +__lockfree_compare_exchange_16(register _odp_u128_t *var, _odp_u128_t *exp, + register _odp_u128_t neu, bool weak, int mo_success, int mo_failure) { (void)weak; /* Always do strong CAS or we can't perform atomic read */ /* Ignore memory ordering for failure, memory order for * success must be stronger or equal. */ (void)mo_failure; - register __int128 old; - register __int128 expected; + register _odp_u128_t old; + register _odp_u128_t expected; int ll_mo = LL_MO(mo_success); int sc_mo = SC_MO(mo_success); @@ -66,10 +63,10 @@ __lockfree_compare_exchange_16(register __int128 *var, __int128 *exp, return old == expected; } -static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu, - int mo) +static inline _odp_u128_t __lockfree_exchange_16(_odp_u128_t *var, + _odp_u128_t neu, int mo) { - register __int128 old; + register _odp_u128_t old; int ll_mo = LL_MO(mo); int sc_mo = SC_MO(mo); @@ -81,10 +78,10 @@ static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu, return old; } -static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask, - int mo) +static inline _odp_u128_t __lockfree_fetch_and_16(_odp_u128_t *var, + _odp_u128_t mask, int mo) { - register __int128 old; + register _odp_u128_t old; int ll_mo = LL_MO(mo); int sc_mo = SC_MO(mo); @@ -96,10 +93,10 @@ static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask, return old; } -static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask, - int mo) +static inline _odp_u128_t __lockfree_fetch_or_16(_odp_u128_t *var, + _odp_u128_t mask, int mo) { - register __int128 old; + register _odp_u128_t old; int ll_mo = LL_MO(mo); int sc_mo = SC_MO(mo); @@ -113,8 +110,8 @@ static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask, #else -static inline __int128_t cas_u128(__int128_t *ptr, __int128_t old_val, - __int128_t new_val, int mo) +static inline _odp_u128_t cas_u128(_odp_u128_t *ptr, _odp_u128_t old_val, + _odp_u128_t new_val, int mo) { /* CASP instructions require that the first register number is paired */ register uint64_t old0 __asm__ ("x0"); @@ -151,18 +148,18 @@ static inline __int128_t cas_u128(__int128_t *ptr, __int128_t old_val, abort(); } - return ((__int128)old0) | (((__int128)old1) << 64); + return ((_odp_u128_t)old0) | (((_odp_u128_t)old1) << 64); } static inline bool -__lockfree_compare_exchange_16(register __int128 *var, __int128 *exp, - register __int128 neu, bool weak, int mo_success, +__lockfree_compare_exchange_16(register _odp_u128_t *var, _odp_u128_t *exp, + register _odp_u128_t neu, bool weak, int mo_success, int mo_failure) { (void)weak; (void)mo_failure; - __int128 old; - __int128 expected; + _odp_u128_t old; + _odp_u128_t expected; expected = *exp; old = cas_u128(var, expected, neu, mo_success); @@ -170,11 +167,11 @@ __lockfree_compare_exchange_16(register __int128 *var, __int128 *exp, return old == expected; } -static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu, - int mo) +static inline _odp_u128_t __lockfree_exchange_16(_odp_u128_t *var, + _odp_u128_t neu, int mo) { - __int128 old; - __int128 expected; + _odp_u128_t old; + _odp_u128_t expected; do { expected = *var; @@ -183,11 +180,11 @@ static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu, return old; } -static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask, - int mo) +static inline _odp_u128_t __lockfree_fetch_and_16(_odp_u128_t *var, + _odp_u128_t mask, int mo) { - __int128 old; - __int128 expected; + _odp_u128_t old; + _odp_u128_t expected; do { expected = *var; @@ -196,11 +193,11 @@ static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask, return old; } -static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask, - int mo) +static inline _odp_u128_t __lockfree_fetch_or_16(_odp_u128_t *var, + _odp_u128_t mask, int mo) { - __int128 old; - __int128 expected; + _odp_u128_t old; + _odp_u128_t expected; do { expected = *var; @@ -211,9 +208,9 @@ static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask, #endif /* __ARM_FEATURE_QRDMX */ -static inline __int128 __lockfree_load_16(__int128 *var, int mo) +static inline _odp_u128_t __lockfree_load_16(_odp_u128_t *var, int mo) { - __int128 old = *var; /* Possibly torn read */ + _odp_u128_t old = *var; /* Possibly torn read */ /* Do CAS to ensure atomicity * Either CAS succeeds (writing back the same value) @@ -223,19 +220,17 @@ static inline __int128 __lockfree_load_16(__int128 *var, int mo) return old; } -__extension__ typedef unsigned __int128 _u128_t; - -static inline _u128_t lockfree_load_u128(_u128_t *atomic) +static inline _odp_u128_t lockfree_load_u128(_odp_u128_t *atomic) { - return __lockfree_load_16((__int128 *)atomic, __ATOMIC_RELAXED); + return __lockfree_load_16((_odp_u128_t *)atomic, __ATOMIC_RELAXED); } -static inline int lockfree_cas_acq_rel_u128(_u128_t *atomic, - _u128_t old_val, - _u128_t new_val) +static inline int lockfree_cas_acq_rel_u128(_odp_u128_t *atomic, + _odp_u128_t old_val, + _odp_u128_t new_val) { - return __lockfree_compare_exchange_16((__int128 *)atomic, - (__int128 *)&old_val, + return __lockfree_compare_exchange_16((_odp_u128_t *)atomic, + (_odp_u128_t *)&old_val, new_val, 0, __ATOMIC_ACQ_REL, @@ -249,7 +244,7 @@ static inline int lockfree_check_u128(void) /** Atomic bit set operations with memory ordering */ #if defined(__SIZEOF_INT128__) && __SIZEOF_INT128__ == 16 -typedef __int128 bitset_t; +typedef _odp_u128_t bitset_t; #define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT128__) #elif __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \ @@ -292,11 +287,9 @@ static inline bitset_t bitset_mask(uint32_t bit) if (bit < 64) return 1ULL << bit; else - return (unsigned __int128)(1ULL << (bit - 64)) << 64; + return (_odp_u128_t)(1ULL << (bit - 64)) << 64; } -#pragma GCC diagnostic pop - #else #error Unsupported size of bit sets (ATOM_BITSET_SIZE) #endif diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c index d719c6cfa..4d46846ce 100644 --- a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c +++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c @@ -1,5 +1,6 @@ /* Copyright (c) 2014-2018, Linaro Limited * Copyright (c) 2021, ARM Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -37,9 +38,9 @@ /* * ARM crypto library may read up to 15 bytes past the end of input - * data and AAD. + * data and AAD and write up to 15 bytes past the end of output data. */ -#define OOB_READ_LEN 15 +#define OOB_WRITE_LEN 16 /* rounded up to 16 bytes for efficiency */ /* * Data buffer size must be a multiple of 16, because the ARM crypto @@ -80,9 +81,11 @@ static const odp_crypto_cipher_capability_t cipher_capa_aes_gcm[] = { static const odp_crypto_auth_capability_t auth_capa_null[] = { {.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; +#define AES_GCM_TAG_LEN 16 + #ifdef __ARM_FEATURE_AES static const odp_crypto_auth_capability_t auth_capa_aes_gcm[] = { -{.digest_len = 16, .key_len = 0, .aad_len = {.min = 8, .max = 12, .inc = 4} } }; +{.digest_len = AES_GCM_TAG_LEN, .key_len = 0, .aad_len = {.min = 8, .max = 12, .inc = 4} } }; #endif /** Forward declaration of session structure */ @@ -92,9 +95,9 @@ typedef struct odp_crypto_generic_session_t odp_crypto_generic_session_t; * Algorithm handler function prototype */ typedef -odp_crypto_alg_err_t (*crypto_func_t)(odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param, - odp_crypto_generic_session_t *session); +odp_bool_t (*crypto_func_t)(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session); /** * Per crypto session data structure @@ -105,15 +108,12 @@ struct odp_crypto_generic_session_t { /* Session creation parameters */ odp_crypto_session_param_t p; - odp_bool_t do_cipher_first; - struct { #if ODP_DEPRECATED_API /* Copy of session IV data */ uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH]; #endif uint8_t key_data[ARM_CRYPTO_MAX_CIPHER_KEY_LENGTH]; - crypto_func_t func; } cipher; struct { @@ -121,11 +121,10 @@ struct odp_crypto_generic_session_t { #if ODP_DEPRECATED_API uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH]; #endif - crypto_func_t func; } auth; + crypto_func_t func; unsigned int idx; - armv8_cipher_constants_t cc; }; @@ -141,7 +140,6 @@ static odp_crypto_global_t *global; typedef struct crypto_local_t { uint8_t buffer[ARM_CRYPTO_MAX_DATA_LENGTH]; - uint8_t digest[ARM_CRYPTO_MAX_DIGEST_LENGTH]; } crypto_local_t; static __thread crypto_local_t local; @@ -176,18 +174,60 @@ void free_session(odp_crypto_generic_session_t *session) odp_spinlock_unlock(&global->lock); } -static odp_crypto_alg_err_t +static +odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) +{ + odp_packet_hdr_t *hdr = packet_hdr(pkt); + + return &hdr->crypto_op_result; +} + +static inline void set_crypto_op_result(odp_packet_t pkt, + odp_crypto_alg_err_t cipher_err, + odp_crypto_alg_err_t auth_err) +{ + odp_crypto_packet_result_t *op_result; + + op_result = get_op_result_from_packet(pkt); + op_result->cipher_status.alg_err = cipher_err; + op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; + op_result->auth_status.alg_err = auth_err; + op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; + op_result->ok = (cipher_err == ODP_CRYPTO_ALG_ERR_NONE && + auth_err == ODP_CRYPTO_ALG_ERR_NONE); +} + +static inline void set_crypto_op_result_ok(odp_packet_t pkt) +{ + set_crypto_op_result(pkt, + ODP_CRYPTO_ALG_ERR_NONE, + ODP_CRYPTO_ALG_ERR_NONE); +} + +static odp_bool_t null_crypto_routine(odp_packet_t pkt ODP_UNUSED, const odp_crypto_packet_op_param_t *param ODP_UNUSED, odp_crypto_generic_session_t *session ODP_UNUSED) { - return ODP_CRYPTO_ALG_ERR_NONE; + set_crypto_op_result_ok(pkt); + return true; +} + +static inline void copy_aad(uint8_t *dst, uint8_t *src, uint32_t len) +{ + ODP_ASSERT(len == 8 || len == 12); + + /* Use constant length memcpy for better optimization result */ + if (len == 8) + memcpy(dst, src, 8); + else + memcpy(dst, src, 12); } static -odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param, - odp_crypto_generic_session_t *session) +odp_bool_t aes_gcm_encrypt(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) { armv8_cipher_state_t cs = { .counter = { @@ -200,12 +240,14 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt, uint64_t aad_bit_length = session->p.auth_aad_len * 8; uint32_t in_pos = param->cipher_range.offset; uint32_t in_len = param->cipher_range.length; - int ret = 0; + odp_bool_t continuous_data; + uint16_t saved_tail[OOB_WRITE_LEN]; + uint8_t tag[AES_GCM_TAG_LEN]; + int rc; /* Fail early if cipher_range is too large */ - if (in_len > ARM_CRYPTO_MAX_DATA_LENGTH) { + if (odp_unlikely(in_len > ARM_CRYPTO_MAX_DATA_LENGTH)) { ODP_DBG("ARM Crypto: Packet size too large for requested operation\n"); - ret = -1; goto err; } @@ -223,9 +265,9 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt, cs.constants = &session->cc; - if (armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs) != 0) { + rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs); + if (odp_unlikely(rc)) { ODP_DBG("ARM Crypto: Failure while setting nonce\n"); - ret = -1; goto err; } @@ -233,46 +275,60 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt, * read it in 16 byte chunks. */ uint8_t aad[ARM_CRYPTO_MAX_AAD_LENGTH]; - memcpy(aad, param->aad_ptr, session->p.auth_aad_len); + copy_aad(aad, param->aad_ptr, session->p.auth_aad_len); uint32_t seg_len = 0; uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL); if (odp_unlikely(odp_packet_is_segmented(pkt)) || - odp_unlikely(odp_packet_tailroom(pkt) < OOB_READ_LEN)) { - /* Packet is segmented or it may not be safe to read beyond - * the end of packet data. Copy the cipher range to a + odp_unlikely(odp_packet_tailroom(pkt) < OOB_WRITE_LEN)) { + /* Packet is segmented or it may not be safe to read and write + * beyond the end of packet data. Copy the cipher range to a * contiguous buffer. */ odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer); data = local.buffer; + continuous_data = false; + } else { + /* Save data that might get overwritten */ + memcpy(saved_tail, data + in_len, OOB_WRITE_LEN); + continuous_data = true; } - if (armv8_enc_aes_gcm_from_state(&cs, - aad, aad_bit_length, - data, plaintext_bit_length, - local.buffer, - local.digest) != 0) { + rc = armv8_enc_aes_gcm_from_state(&cs, + aad, aad_bit_length, + data, plaintext_bit_length, + data, + tag); + if (odp_unlikely(rc)) { ODP_DBG("ARM Crypto: AES GCM Encoding failed\n"); - ret = -1; goto err; } - odp_packet_copy_from_mem(pkt, param->cipher_range.offset, - param->cipher_range.length, local.buffer); + if (odp_likely(continuous_data)) { + memcpy(data + in_len, saved_tail, OOB_WRITE_LEN); + memcpy(data - in_pos + param->hash_result_offset, + tag, AES_GCM_TAG_LEN); + } else { + odp_packet_copy_from_mem(pkt, in_pos, in_len, data); + odp_packet_copy_from_mem(pkt, param->hash_result_offset, + AES_GCM_TAG_LEN, tag); + } - odp_packet_copy_from_mem(pkt, param->hash_result_offset, - session->p.auth_digest_len, local.digest); + set_crypto_op_result_ok(pkt); + return true; err: - return ret < 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : - ODP_CRYPTO_ALG_ERR_NONE; + set_crypto_op_result(pkt, + ODP_CRYPTO_ALG_ERR_DATA_SIZE, + ODP_CRYPTO_ALG_ERR_NONE); + return false; } static -odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param, - odp_crypto_generic_session_t *session) +odp_bool_t aes_gcm_decrypt(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) { armv8_cipher_state_t cs = { .counter = { @@ -280,18 +336,19 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt, } }; uint8_t *iv_ptr; - uint8_t tag[16]; + uint8_t tag[AES_GCM_TAG_LEN]; uint64_t iv_bit_length = session->p.cipher_iv_len * 8; uint64_t plaintext_bit_length = param->cipher_range.length * 8; uint64_t aad_bit_length = session->p.auth_aad_len * 8; uint32_t in_pos = param->cipher_range.offset; uint32_t in_len = param->cipher_range.length; - int ret = 0; + odp_bool_t continuous_data; + uint16_t saved_tail[OOB_WRITE_LEN]; + int rc; /* Fail early if cipher_range is too large */ - if (in_len > ARM_CRYPTO_MAX_DATA_LENGTH) { + if (odp_unlikely(in_len > ARM_CRYPTO_MAX_DATA_LENGTH)) { ODP_DBG("ARM Crypto: Packet size too large for requested operation\n"); - ret = -1; goto err; } @@ -309,51 +366,63 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt, cs.constants = &session->cc; - if (armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs) != 0) { + rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs); + if (odp_unlikely(rc)) { ODP_DBG("ARM Crypto: Failure while setting nonce\n"); - ret = -1; goto err; } - /* Copy current hash bytes to a buffer and clear it */ - odp_packet_copy_to_mem(pkt, param->hash_result_offset, - session->p.auth_digest_len, tag); - /* Copy AAD in a stack to make sure that the ARM crypto library can * read it in 16 byte chunks. */ uint8_t aad[ARM_CRYPTO_MAX_AAD_LENGTH]; - memcpy(aad, param->aad_ptr, session->p.auth_aad_len); + copy_aad(aad, param->aad_ptr, session->p.auth_aad_len); uint32_t seg_len = 0; uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL); if (odp_unlikely(odp_packet_is_segmented(pkt)) || - odp_unlikely(odp_packet_tailroom(pkt) < OOB_READ_LEN)) { - /* Packet is segmented or it may not be safe to read beyond - * the end of packet data. Copy the cipher range to a + odp_unlikely(odp_packet_tailroom(pkt) < OOB_WRITE_LEN)) { + /* Packet is segmented or it may not be safe to read and write + * beyond the end of packet data. Copy the cipher range to a * contiguous buffer. */ odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer); - data = local.buffer; + /* Copy tag from the packet to a buffer */ + odp_packet_copy_to_mem(pkt, param->hash_result_offset, + AES_GCM_TAG_LEN, tag); + continuous_data = false; + } else { + /* Save data that might get overwritten */ + memcpy(saved_tail, data + in_len, OOB_WRITE_LEN); + /* Copy tag from the packet to a buffer */ + memcpy(tag, data - in_pos + param->hash_result_offset, AES_GCM_TAG_LEN); + continuous_data = true; } - if (armv8_dec_aes_gcm_from_state(&cs, - aad, aad_bit_length, - data, plaintext_bit_length, - tag, - local.buffer) != 0) { + rc = armv8_dec_aes_gcm_from_state(&cs, + aad, aad_bit_length, + data, plaintext_bit_length, + tag, + data); + if (odp_unlikely(rc)) { ODP_DBG("ARM Crypto: AES GCM Decoding failed\n"); - ret = -1; goto err; } - odp_packet_copy_from_mem(pkt, param->cipher_range.offset, - param->cipher_range.length, local.buffer); + if (odp_likely(continuous_data)) + memcpy(data + in_len, saved_tail, OOB_WRITE_LEN); + else + odp_packet_copy_from_mem(pkt, in_pos, in_len, data); + + set_crypto_op_result_ok(pkt); + return true; err: - return ret < 0 ? ODP_CRYPTO_ALG_ERR_ICV_CHECK : - ODP_CRYPTO_ALG_ERR_NONE; + set_crypto_op_result(pkt, + ODP_CRYPTO_ALG_ERR_NONE, + ODP_CRYPTO_ALG_ERR_ICV_CHECK); + return false; } static int process_aes_gcm_param(odp_crypto_generic_session_t *session) @@ -376,9 +445,9 @@ static int process_aes_gcm_param(odp_crypto_generic_session_t *session) /* Set function */ if (ODP_CRYPTO_OP_ENCODE == session->p.op) - session->cipher.func = aes_gcm_encrypt; + session->func = aes_gcm_encrypt; else - session->cipher.func = aes_gcm_decrypt; + session->func = aes_gcm_decrypt; return 0; } @@ -520,16 +589,10 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, session->p.auth_iv.length); #endif - /* Derive order */ - if (ODP_CRYPTO_OP_ENCODE == param->op) - session->do_cipher_first = param->auth_cipher_text; - else - session->do_cipher_first = !param->auth_cipher_text; - /* Process based on cipher */ switch (param->cipher_alg) { case ODP_CIPHER_ALG_NULL: - session->cipher.func = null_crypto_routine; + session->func = null_crypto_routine; rc = 0; break; case ODP_CIPHER_ALG_AES_GCM: @@ -584,14 +647,15 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, /* Process based on auth */ switch (param->auth_alg) { case ODP_AUTH_ALG_NULL: - session->auth.func = null_crypto_routine; - rc = 0; + if (param->cipher_alg == ODP_CIPHER_ALG_NULL) + rc = 0; + else + rc = -1; break; case ODP_AUTH_ALG_AES_GCM: /* AES-GCM requires to do both auth and * cipher at the same time */ if (param->cipher_alg == ODP_CIPHER_ALG_AES_GCM) { - session->auth.func = null_crypto_routine; rc = 0; } else { rc = -1; @@ -630,9 +694,7 @@ int odp_crypto_session_destroy(odp_crypto_session_t session) return 0; } -/* - * Shim function around packet operation, can be used by other implementations. - */ +#if ODP_DEPRECATED_API int odp_crypto_operation(odp_crypto_op_param_t *param, odp_bool_t *posted, @@ -687,6 +749,7 @@ odp_crypto_operation(odp_crypto_op_param_t *param, return 0; } +#endif int _odp_crypto_init_global(void) { @@ -767,6 +830,7 @@ int _odp_crypto_term_local(void) return 0; } +#if ODP_DEPRECATED_API odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev) { /* This check not mandated by the API specification */ @@ -803,6 +867,7 @@ uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl) { return _odp_pri(hdl); } +#endif /* ODP_DEPRECATED_API */ void odp_crypto_session_param_init(odp_crypto_session_param_t *param) { @@ -828,14 +893,6 @@ odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) return odp_packet_to_event(pkt); } -static -odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) -{ - odp_packet_hdr_t *hdr = packet_hdr(pkt); - - return &hdr->crypto_op_result; -} - int odp_crypto_result(odp_crypto_packet_result_t *result, odp_packet_t packet) { @@ -856,18 +913,16 @@ int crypto_int(odp_packet_t pkt_in, odp_packet_t *pkt_out, const odp_crypto_packet_op_param_t *param) { - odp_crypto_alg_err_t rc_cipher = ODP_CRYPTO_ALG_ERR_NONE; - odp_crypto_alg_err_t rc_auth = ODP_CRYPTO_ALG_ERR_NONE; odp_crypto_generic_session_t *session; odp_bool_t allocated = false; odp_packet_t out_pkt = *pkt_out; - odp_crypto_packet_result_t *op_result; odp_packet_hdr_t *pkt_hdr; + odp_bool_t ok; session = (odp_crypto_generic_session_t *)(intptr_t)param->session; /* Resolve output buffer */ - if (ODP_PACKET_INVALID == out_pkt && + if (odp_unlikely(ODP_PACKET_INVALID == out_pkt) && ODP_POOL_INVALID != session->p.output_pool) { out_pkt = odp_packet_alloc(session->p.output_pool, odp_packet_len(pkt_in)); @@ -879,7 +934,7 @@ int crypto_int(odp_packet_t pkt_in, return -1; } - if (pkt_in != out_pkt) { + if (odp_unlikely(pkt_in != out_pkt)) { int ret; int md_copy; @@ -903,28 +958,12 @@ int crypto_int(odp_packet_t pkt_in, pkt_in = ODP_PACKET_INVALID; } - /* Invoke the functions */ - if (session->do_cipher_first) { - rc_cipher = session->cipher.func(out_pkt, param, session); - rc_auth = session->auth.func(out_pkt, param, session); - } else { - rc_auth = session->auth.func(out_pkt, param, session); - rc_cipher = session->cipher.func(out_pkt, param, session); - } + /* Invoke the crypto function */ + ok = session->func(out_pkt, param, session); - /* Fill in result */ packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO); - op_result = get_op_result_from_packet(out_pkt); - op_result->cipher_status.alg_err = rc_cipher; - op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; - op_result->auth_status.alg_err = rc_auth; - op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; - op_result->ok = - (rc_cipher == ODP_CRYPTO_ALG_ERR_NONE) && - (rc_auth == ODP_CRYPTO_ALG_ERR_NONE); - pkt_hdr = packet_hdr(out_pkt); - pkt_hdr->p.flags.crypto_err = !op_result->ok; + pkt_hdr->p.flags.crypto_err = !ok; /* Synchronous, simply return results */ *pkt_out = out_pkt; diff --git a/platform/linux-generic/arch/aarch64/odp_llsc.h b/platform/linux-generic/arch/aarch64/odp_llsc.h index 2561451ca..1c2610511 100644 --- a/platform/linux-generic/arch/aarch64/odp_llsc.h +++ b/platform/linux-generic/arch/aarch64/odp_llsc.h @@ -13,6 +13,8 @@ #error This file should not be included directly, please include odp_cpu.h #endif +#include <odp_types_internal.h> + static inline uint16_t ll8(uint8_t *var, int mm) { uint16_t old; @@ -115,11 +117,11 @@ static inline uint32_t sc(uint64_t *var, uint64_t neu, int mm) #define sc64(a, b, c) sc((a), (b), (c)) union i128 { - __extension__ __int128 i128; + _odp_u128_t i128; int64_t i64[2]; }; -__extension__ static inline __int128 lld(__int128 *var, int mm) +static inline _odp_u128_t lld(_odp_u128_t *var, int mm) { union i128 old; @@ -139,7 +141,7 @@ __extension__ static inline __int128 lld(__int128 *var, int mm) } /* Return 0 on success, 1 on failure */ -__extension__ static inline uint32_t scd(__int128 *var, __int128 neu, int mm) +static inline uint32_t scd(_odp_u128_t *var, _odp_u128_t neu, int mm) { uint32_t ret; diff --git a/platform/linux-generic/arch/arm/odp_atomic.h b/platform/linux-generic/arch/arm/odp_atomic.h index bcc89ff73..e400f52d4 100644 --- a/platform/linux-generic/arch/arm/odp_atomic.h +++ b/platform/linux-generic/arch/arm/odp_atomic.h @@ -12,6 +12,7 @@ #error This file should not be included directly, please include odp_cpu.h #endif +#include <odp_types_internal.h> #include <limits.h> #ifdef CONFIG_DMBSTR @@ -70,7 +71,7 @@ static inline bitset_t bitset_mask(uint32_t bit) if (bit < 64) return 1ULL << bit; else - return (unsigned __int128)(1ULL << (bit - 64)) << 64; + return (_odp_u128_t)(1ULL << (bit - 64)) << 64; } #else diff --git a/platform/linux-generic/arch/default/odp_atomic.h b/platform/linux-generic/arch/default/odp_atomic.h index 7650d2b52..4cfc6b4bd 100644 --- a/platform/linux-generic/arch/default/odp_atomic.h +++ b/platform/linux-generic/arch/default/odp_atomic.h @@ -7,18 +7,18 @@ #ifndef ODP_DEFAULT_ATOMIC_H_ #define ODP_DEFAULT_ATOMIC_H_ -#ifdef __SIZEOF_INT128__ +#include <odp_types_internal.h> -__extension__ typedef unsigned __int128 _u128_t; +#ifdef __SIZEOF_INT128__ -static inline _u128_t lockfree_load_u128(_u128_t *atomic) +static inline _odp_u128_t lockfree_load_u128(_odp_u128_t *atomic) { return __atomic_load_n(atomic, __ATOMIC_RELAXED); } -static inline int lockfree_cas_acq_rel_u128(_u128_t *atomic, - _u128_t old_val, - _u128_t new_val) +static inline int lockfree_cas_acq_rel_u128(_odp_u128_t *atomic, + _odp_u128_t old_val, + _odp_u128_t new_val) { return __atomic_compare_exchange_n(atomic, &old_val, new_val, 0 /* strong */, @@ -76,7 +76,7 @@ static inline bitset_t bitset_mask(uint32_t bit) if (bit < 64) return 1ULL << bit; else - return (unsigned __int128)(1ULL << (bit - 64)) << 64; + return (_odp_u128_t)(1ULL << (bit - 64)) << 64; } #else diff --git a/platform/linux-generic/include-abi/odp/api/abi/event.h b/platform/linux-generic/include-abi/odp/api/abi/event.h index 6530ac2e4..e059f318c 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/event.h +++ b/platform/linux-generic/include-abi/odp/api/abi/event.h @@ -1,4 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -18,42 +19,9 @@ extern "C" { #endif -#include <odp/api/plat/strong_types.h> - -/** @ingroup odp_event - * @{ - */ - -typedef ODP_HANDLE_T(odp_event_t); - -#define ODP_EVENT_INVALID _odp_cast_scalar(odp_event_t, 0) - -typedef enum odp_event_type_t { - ODP_EVENT_BUFFER = 1, - ODP_EVENT_PACKET = 2, - ODP_EVENT_TIMEOUT = 3, - ODP_EVENT_CRYPTO_COMPL = 4, - ODP_EVENT_IPSEC_STATUS = 5, - ODP_EVENT_PACKET_VECTOR = 6, - ODP_EVENT_PACKET_TX_COMPL = 7, - ODP_EVENT_DMA_COMPL = 8, -} odp_event_type_t; - -typedef enum odp_event_subtype_t { - ODP_EVENT_NO_SUBTYPE = 0, - ODP_EVENT_PACKET_BASIC = 1, - ODP_EVENT_PACKET_CRYPTO = 2, - ODP_EVENT_PACKET_IPSEC = 3, - ODP_EVENT_PACKET_COMP = 4 -} odp_event_subtype_t; - /* Inlined functions for non-ABI compat mode */ #include <odp/api/plat/event_inlines.h> -/** - * @} - */ - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include-abi/odp/api/abi/event_types.h b/platform/linux-generic/include-abi/odp/api/abi/event_types.h new file mode 100644 index 000000000..c39322330 --- /dev/null +++ b/platform/linux-generic/include-abi/odp/api/abi/event_types.h @@ -0,0 +1,61 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP event type definitions + */ + +#ifndef ODP_API_ABI_EVENT_TYPES_H_ +#define ODP_API_ABI_EVENT_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/plat/strong_types.h> +#include <odp/api/deprecated.h> + +/** @ingroup odp_event + * @{ + */ + +typedef ODP_HANDLE_T(odp_event_t); + +#define ODP_EVENT_INVALID _odp_cast_scalar(odp_event_t, 0) + +typedef enum odp_event_type_t { + ODP_EVENT_BUFFER = 1, + ODP_EVENT_PACKET = 2, + ODP_EVENT_TIMEOUT = 3, +#if ODP_DEPRECATED_API + ODP_EVENT_CRYPTO_COMPL = 4, +#endif + ODP_EVENT_IPSEC_STATUS = 5, + ODP_EVENT_PACKET_VECTOR = 6, + ODP_EVENT_PACKET_TX_COMPL = 7, + ODP_EVENT_DMA_COMPL = 8, +} odp_event_type_t; + +typedef enum odp_event_subtype_t { + ODP_EVENT_NO_SUBTYPE = 0, + ODP_EVENT_PACKET_BASIC = 1, + ODP_EVENT_PACKET_CRYPTO = 2, + ODP_EVENT_PACKET_IPSEC = 3, + ODP_EVENT_PACKET_COMP = 4 +} odp_event_subtype_t; + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include-abi/odp/api/abi/timer.h b/platform/linux-generic/include-abi/odp/api/abi/timer.h new file mode 100644 index 000000000..0a3b3a9cc --- /dev/null +++ b/platform/linux-generic/include-abi/odp/api/abi/timer.h @@ -0,0 +1,8 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/timer_inlines.h> diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h index 632ba0e69..9e0ae3f50 100644 --- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h @@ -8,7 +8,7 @@ #define ODP_PLAT_BUFFER_INLINES_H_ #include <odp/api/abi/buffer.h> -#include <odp/api/abi/event.h> +#include <odp/api/abi/event_types.h> #include <odp/api/plat/buffer_inline_types.h> diff --git a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h index 6eb34a39b..04ad9e968 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h @@ -1,4 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -30,11 +31,32 @@ static inline uint64_t _odp_packet_input_flags(odp_packet_t pkt) /* Inline functions by default */ #define _ODP_INLINE static inline #define odp_packet_has_l2 __odp_packet_has_l2 + #define odp_packet_has_l3 __odp_packet_has_l3 + #define odp_packet_has_l4 __odp_packet_has_l4 #define odp_packet_has_eth __odp_packet_has_eth #define odp_packet_has_jumbo __odp_packet_has_jumbo #define odp_packet_has_flow_hash __odp_packet_has_flow_hash #define odp_packet_has_ts __odp_packet_has_ts #define odp_packet_has_ipsec __odp_packet_has_ipsec + #define odp_packet_has_eth_bcast __odp_packet_has_eth_bcast + #define odp_packet_has_eth_mcast __odp_packet_has_eth_mcast + #define odp_packet_has_vlan __odp_packet_has_vlan + #define odp_packet_has_vlan_qinq __odp_packet_has_vlan_qinq + #define odp_packet_has_arp __odp_packet_has_arp + #define odp_packet_has_ipv4 __odp_packet_has_ipv4 + #define odp_packet_has_ipv6 __odp_packet_has_ipv6 + #define odp_packet_has_ip_bcast __odp_packet_has_ip_bcast + #define odp_packet_has_ip_mcast __odp_packet_has_ip_mcast + #define odp_packet_has_ipfrag __odp_packet_has_ipfrag + #define odp_packet_has_ipopt __odp_packet_has_ipopt + #define odp_packet_has_udp __odp_packet_has_udp + #define odp_packet_has_tcp __odp_packet_has_tcp + #define odp_packet_has_sctp __odp_packet_has_sctp + #define odp_packet_has_icmp __odp_packet_has_icmp + #define odp_packet_has_error __odp_packet_has_error + #define odp_packet_has_l2_error __odp_packet_has_l2_error + #define odp_packet_has_l3_error __odp_packet_has_l3_error + #define odp_packet_has_l4_error __odp_packet_has_l4_error #else #undef _ODP_INLINE #define _ODP_INLINE @@ -48,6 +70,22 @@ _ODP_INLINE int odp_packet_has_l2(odp_packet_t pkt) return flags.l2; } +_ODP_INLINE int odp_packet_has_l3(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.l3; +} + +_ODP_INLINE int odp_packet_has_l4(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.l4; +} + _ODP_INLINE int odp_packet_has_eth(odp_packet_t pkt) { _odp_packet_input_flags_t flags; @@ -88,6 +126,163 @@ _ODP_INLINE int odp_packet_has_ipsec(odp_packet_t pkt) return flags.ipsec; } +_ODP_INLINE int odp_packet_has_eth_bcast(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.eth_bcast; +} + +_ODP_INLINE int odp_packet_has_eth_mcast(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.eth_mcast; +} + +_ODP_INLINE int odp_packet_has_vlan(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.vlan; +} + +_ODP_INLINE int odp_packet_has_vlan_qinq(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.vlan_qinq; +} + +_ODP_INLINE int odp_packet_has_arp(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.arp; +} + +_ODP_INLINE int odp_packet_has_ipv4(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.ipv4; +} + +_ODP_INLINE int odp_packet_has_ipv6(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.ipv6; +} + +_ODP_INLINE int odp_packet_has_ip_bcast(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.ip_bcast; +} + +_ODP_INLINE int odp_packet_has_ip_mcast(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.ip_mcast; +} + +_ODP_INLINE int odp_packet_has_ipfrag(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.ipfrag; +} + +_ODP_INLINE int odp_packet_has_ipopt(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.ipopt; +} + +_ODP_INLINE int odp_packet_has_udp(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.udp; +} + +_ODP_INLINE int odp_packet_has_tcp(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.tcp; +} + +_ODP_INLINE int odp_packet_has_sctp(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.sctp; +} + +_ODP_INLINE int odp_packet_has_icmp(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.icmp; +} + +_ODP_INLINE int odp_packet_has_error(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + return flags.all.error != 0; +} + +_ODP_INLINE int odp_packet_has_l2_error(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + + /* L2 parsing is always done by default and hence + no additional check is required. */ + return flags.snap_len_err; +} + +_ODP_INLINE int odp_packet_has_l3_error(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + + return flags.ip_err; +} + +_ODP_INLINE int odp_packet_has_l4_error(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + + return flags.tcp_err | flags.udp_err; +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index 66451ce49..126404f2d 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -115,7 +115,7 @@ typedef union { uint32_t all_flags; struct { - uint32_t reserved1: 7; + uint32_t reserved1: 6; /* * Init flags @@ -133,6 +133,7 @@ typedef union { uint32_t l4_chksum: 1; /* L4 chksum override */ uint32_t ts_set: 1; /* Set Tx timestamp */ uint32_t tx_compl: 1; /* Tx completion event requested */ + uint32_t tx_aging: 1; /* Packet aging at Tx requested */ uint32_t shaper_len_adj: 8; /* Adjustment for traffic mgr */ /* @@ -150,8 +151,8 @@ typedef union { /* Flag groups */ struct { - uint32_t reserved2: 7; - uint32_t other: 17; /* All other flags */ + uint32_t reserved2: 6; + uint32_t other: 18; /* All other flags */ uint32_t error: 8; /* All error flags */ } all; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index fb63f207b..b72749ea8 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -20,7 +20,7 @@ #include <odp/api/time.h> #include <odp/api/abi/buffer.h> -#include <odp/api/abi/event.h> +#include <odp/api/abi/event_types.h> #include <odp/api/abi/packet.h> #include <odp/api/abi/packet_io.h> @@ -73,6 +73,9 @@ #define odp_packet_subtype __odp_packet_subtype #define odp_packet_tx_compl_from_event __odp_packet_tx_compl_from_event #define odp_packet_tx_compl_to_event __odp_packet_tx_compl_to_event + #define odp_packet_color __odp_packet_color + #define odp_packet_drop_eligible __odp_packet_drop_eligible + #define odp_packet_shaper_len_adjust __odp_packet_shaper_len_adjust #else #undef _ODP_INLINE #define _ODP_INLINE @@ -372,6 +375,33 @@ _ODP_INLINE odp_event_t odp_packet_tx_compl_to_event(odp_packet_tx_compl_t tx_co return (odp_event_t)(uintptr_t)tx_compl; } +_ODP_INLINE odp_packet_color_t odp_packet_color(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + return (odp_packet_color_t)input_flags.color; +} + +_ODP_INLINE odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + return !input_flags.nodrop; +} + +_ODP_INLINE int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + + return flags.shaper_len_adj; +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h index 3586aa6d5..54a009a36 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h @@ -14,7 +14,7 @@ #ifndef _ODP_PLAT_PACKET_VECTOR_INLINES_H_ #define _ODP_PLAT_PACKET_VECTOR_INLINES_H_ -#include <odp/api/abi/event.h> +#include <odp/api/abi/event_types.h> #include <odp/api/abi/packet_types.h> #include <odp/api/abi/pool_types.h> diff --git a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h index 5ce767a16..4eade3dea 100644 --- a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h @@ -12,6 +12,7 @@ extern "C" { #endif #include <stdint.h> +#include <odp/api/event_types.h> #include <odp/api/queue_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/platform/linux-generic/include/odp/api/plat/time_inlines.h b/platform/linux-generic/include/odp/api/plat/time_inlines.h index bb2913532..2ffb94c66 100644 --- a/platform/linux-generic/include/odp/api/plat/time_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/time_inlines.h @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2020-2021, Nokia + * Copyright (c) 2020-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -99,6 +99,7 @@ static inline uint64_t _odp_time_convert_to_ns(odp_time_t time) #define odp_time_cmp __odp_time_cmp #define odp_time_diff __odp_time_diff + #define odp_time_diff_ns __odp_time_diff_ns #define odp_time_sum __odp_time_sum #else @@ -170,6 +171,15 @@ _ODP_INLINE odp_time_t odp_time_diff(odp_time_t t2, odp_time_t t1) return time; } +_ODP_INLINE uint64_t odp_time_diff_ns(odp_time_t t2, odp_time_t t1) +{ + odp_time_t time; + + time.u64 = t2.u64 - t1.u64; + + return odp_time_to_ns(time); +} + _ODP_INLINE odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2) { odp_time_t time; diff --git a/platform/linux-generic/include/odp/api/plat/timer_inline_types.h b/platform/linux-generic/include/odp/api/plat/timer_inline_types.h new file mode 100644 index 000000000..e3397c4df --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/timer_inline_types.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_TIMER_INLINE_TYPES_H_ +#define ODP_PLAT_TIMER_INLINE_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +/* Timeout header field accessor */ +#define _odp_timeout_hdr_field(hdr, cast, field) \ + (*(cast *)(uintptr_t)((uint8_t *)hdr + \ + _odp_timeout_inline_offset.field)) + +/* Timeout header field offsets for inline functions */ +typedef struct _odp_timeout_inline_offset_t { + uint16_t expiration; + uint16_t timer; + uint16_t user_ptr; + +} _odp_timeout_inline_offset_t; + +/** @endcond */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/timer_inlines.h b/platform/linux-generic/include/odp/api/plat/timer_inlines.h new file mode 100644 index 000000000..406aefdf6 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/timer_inlines.h @@ -0,0 +1,78 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_TIMER_INLINES_H_ +#define ODP_PLAT_TIMER_INLINES_H_ + +#include <odp/api/event.h> +#include <odp/api/timer_types.h> + +#include <odp/api/plat/timer_inline_types.h> + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +extern const _odp_timeout_inline_offset_t _odp_timeout_inline_offset; + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_timeout_timer __odp_timeout_timer + #define odp_timeout_tick __odp_timeout_tick + #define odp_timeout_user_ptr __odp_timeout_user_ptr + #define odp_timer_tick_to_ns __odp_timer_tick_to_ns + #define odp_timer_ns_to_tick __odp_timer_ns_to_tick + #define odp_timeout_from_event __odp_timeout_from_event + #define odp_timeout_to_event __odp_timeout_to_event +#else + #define _ODP_INLINE +#endif + +_ODP_INLINE odp_timer_t odp_timeout_timer(odp_timeout_t tmo) +{ + return _odp_timeout_hdr_field(tmo, odp_timer_t, timer); +} + +_ODP_INLINE uint64_t odp_timeout_tick(odp_timeout_t tmo) +{ + return _odp_timeout_hdr_field(tmo, uint64_t, expiration); +} + +_ODP_INLINE void *odp_timeout_user_ptr(odp_timeout_t tmo) +{ + return _odp_timeout_hdr_field(tmo, void *, user_ptr); +} + +_ODP_INLINE uint64_t odp_timer_tick_to_ns(odp_timer_pool_t tp, uint64_t ticks) +{ + (void)tp; + + /* Timer ticks in API are nsec */ + return ticks; +} + +_ODP_INLINE uint64_t odp_timer_ns_to_tick(odp_timer_pool_t tp, uint64_t ns) +{ + (void)tp; + + /* Timer ticks in API are nsec */ + return ns; +} + +_ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev) +{ + return (odp_timeout_t)ev; +} + +_ODP_INLINE odp_event_t odp_timeout_to_event(odp_timeout_t tmo) +{ + return (odp_event_t)tmo; +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp_atomic_internal.h b/platform/linux-generic/include/odp_atomic_internal.h index d98e0e6be..da79b3723 100644 --- a/platform/linux-generic/include/odp_atomic_internal.h +++ b/platform/linux-generic/include/odp_atomic_internal.h @@ -20,6 +20,7 @@ #include <odp/api/align.h> #include <odp/api/hints.h> #include <odp/api/atomic.h> +#include <odp_types_internal.h> #include <stdbool.h> #ifdef __cplusplus @@ -137,12 +138,10 @@ static inline void _odp_atomic_flag_clear(_odp_atomic_flag_t *flag) #endif #ifdef ODP_ATOMIC_U128 -/** An unsigned 128-bit (16-byte) scalar type */ -__extension__ typedef __int128 _uint128_t; /** Atomic 128-bit type */ typedef struct ODP_ALIGNED(16) { - _uint128_t v; /**< Actual storage for the atomic variable */ + _odp_u128_t v; /**< Actual storage for the atomic variable */ } _odp_atomic_u128_t; /** @@ -154,8 +153,8 @@ typedef struct ODP_ALIGNED(16) { * @param mmodel Memory model associated with the exchange operation */ static inline void _odp_atomic_u128_xchg_mm(_odp_atomic_u128_t *ptr, - _uint128_t *val, - _uint128_t *old, + _odp_u128_t *val, + _odp_u128_t *old, _odp_memmodel_t mm) { __atomic_exchange(&ptr->v, val, old, mm); @@ -177,8 +176,8 @@ static inline void _odp_atomic_u128_xchg_mm(_odp_atomic_u128_t *ptr, * @retval 0 exchange failed and '*exp' updated with current value */ static inline int _odp_atomic_u128_cmp_xchg_mm(_odp_atomic_u128_t *ptr, - _uint128_t *exp, - _uint128_t *val, + _odp_u128_t *exp, + _odp_u128_t *val, _odp_memmodel_t succ, _odp_memmodel_t fail) { diff --git a/platform/linux-generic/include/odp_classification_datamodel.h b/platform/linux-generic/include/odp_classification_datamodel.h index cc0e7f081..6b50fef68 100644 --- a/platform/linux-generic/include/odp_classification_datamodel.h +++ b/platform/linux-generic/include/odp_classification_datamodel.h @@ -129,6 +129,7 @@ typedef struct pmr_term_value { Class Of Service */ struct cos_s { + odp_cos_action_t action; /* Action */ odp_queue_t queue; /* Associated Queue */ odp_pool_t pool; /* Associated Buffer pool */ odp_pktin_vector_config_t vector; /* Packet vector config */ @@ -145,10 +146,11 @@ struct cos_s { odp_queue_param_t queue_param; char name[ODP_COS_NAME_LEN]; /* name */ uint8_t index; + odp_bool_t stats_enable; struct { odp_atomic_u64_t discards; odp_atomic_u64_t packets; - } stats[CLS_COS_QUEUE_MAX]; + } stats, queue_stats[CLS_COS_QUEUE_MAX]; }; typedef union cos_u { diff --git a/platform/linux-generic/include/odp_classification_internal.h b/platform/linux-generic/include/odp_classification_internal.h index bc0a12f8f..c20d24c58 100644 --- a/platform/linux-generic/include/odp_classification_internal.h +++ b/platform/linux-generic/include/odp_classification_internal.h @@ -67,9 +67,9 @@ static inline void _odp_cos_queue_stats_add(cos_t *cos, odp_queue_t queue, } if (packets) - odp_atomic_add_u64(&cos->s.stats[queue_idx].packets, packets); + odp_atomic_add_u64(&cos->s.queue_stats[queue_idx].packets, packets); if (discards) - odp_atomic_add_u64(&cos->s.stats[queue_idx].discards, discards); + odp_atomic_add_u64(&cos->s.queue_stats[queue_idx].discards, discards); } /** Classification Internal function **/ @@ -84,8 +84,7 @@ This function calls Classifier module internal functions for a given packet and selects destination queue and packet pool based on selected PMR and CoS. **/ int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base, - uint16_t pkt_len, uint32_t seg_len, odp_pool_t *pool, - odp_packet_hdr_t *pkt_hdr, odp_bool_t parse); + odp_pool_t *pool, odp_packet_hdr_t *pkt_hdr); /** Packet IO classifier init diff --git a/platform/linux-generic/include/odp_llqueue.h b/platform/linux-generic/include/odp_llqueue.h index e9cf9945e..68325624a 100644 --- a/platform/linux-generic/include/odp_llqueue.h +++ b/platform/linux-generic/include/odp_llqueue.h @@ -15,6 +15,7 @@ #include <odp_config_internal.h> #include <odp_debug_internal.h> +#include <odp_types_internal.h> #include <odp_cpu.h> #include <stdint.h> @@ -49,7 +50,7 @@ static odp_bool_t llq_on_queue(struct llnode *node); typedef uint64_t dintptr_t; #endif #if __SIZEOF_PTRDIFF_T__ == 8 -__extension__ typedef __int128 dintptr_t; +typedef _odp_u128_t dintptr_t; #endif struct llnode { diff --git a/platform/linux-generic/include/odp_macros_internal.h b/platform/linux-generic/include/odp_macros_internal.h index 997e0fd5b..b8be7f938 100644 --- a/platform/linux-generic/include/odp_macros_internal.h +++ b/platform/linux-generic/include/odp_macros_internal.h @@ -35,6 +35,8 @@ extern "C" { tmp_a > tmp_b ? tmp_a : tmp_b; \ }) +#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c))) + #define odp_container_of(pointer, type, member) \ ((type *)(void *)(((char *)pointer) - offsetof(type, member))) diff --git a/platform/linux-generic/include/odp_packet_dpdk.h b/platform/linux-generic/include/odp_packet_dpdk.h index 3add83f19..b326000e6 100644 --- a/platform/linux-generic/include/odp_packet_dpdk.h +++ b/platform/linux-generic/include/odp_packet_dpdk.h @@ -46,18 +46,4 @@ int _odp_dpdk_packet_parse_common(packet_parser_t *pkt_hdr, uint32_t supported_ptypes, odp_pktin_config_opt_t pktin_cfg); -static inline int _odp_dpdk_packet_parse_layer(odp_packet_hdr_t *pkt_hdr, - struct rte_mbuf *mbuf, - odp_proto_layer_t layer, - uint32_t supported_ptypes, - odp_pktin_config_opt_t pktin_cfg) -{ - uint32_t seg_len = pkt_hdr->seg_len; - void *base = pkt_hdr->seg_data; - - return _odp_dpdk_packet_parse_common(&pkt_hdr->p, base, - pkt_hdr->frame_len, seg_len, mbuf, - layer, supported_ptypes, - pktin_cfg); -} #endif diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 5a29210dd..a8f58146a 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -38,15 +38,23 @@ extern "C" { #include <stdint.h> #include <string.h> -/** Minimum segment length expected by _odp_packet_parse_common() */ -#define PACKET_PARSE_SEG_LEN 96 - ODP_STATIC_ASSERT(sizeof(_odp_packet_input_flags_t) == sizeof(uint64_t), "INPUT_FLAGS_SIZE_ERROR"); ODP_STATIC_ASSERT(sizeof(_odp_packet_flags_t) == sizeof(uint32_t), "PACKET_FLAGS_SIZE_ERROR"); +/* Packet extra data length */ +#define PKT_EXTRA_LEN 128 + +/* Packet extra data types */ +#define PKT_EXTRA_TYPE_DPDK 1 + +/* Maximum number of segments per packet */ +#define PKT_MAX_SEGS 255 + +ODP_STATIC_ASSERT(PKT_MAX_SEGS < UINT16_MAX, "PACKET_MAX_SEGS_ERROR"); + /** * Packet parser metadata */ @@ -67,17 +75,6 @@ typedef struct { uint16_t l4_offset; } packet_parser_t; -/* Packet extra data length */ -#define PKT_EXTRA_LEN 128 - -/* Packet extra data types */ -#define PKT_EXTRA_TYPE_DPDK 1 - -/* Maximum number of segments per packet */ -#define PKT_MAX_SEGS 255 - -ODP_STATIC_ASSERT(PKT_MAX_SEGS < UINT16_MAX, "PACKET_MAX_SEGS_ERROR"); - /** * Internal Packet header * @@ -146,6 +143,10 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t { /* Max payload size in a LSO segment */ uint16_t lso_max_payload; + /* Packet aging drop timeout before enqueue. Once enqueued holds the maximum age (time of + * request + requested drop timeout). */ + uint64_t tx_aging_ns; + /* LSO profile index */ uint8_t lso_profile_idx; @@ -194,6 +195,11 @@ static inline odp_packet_t packet_from_event_hdr(_odp_event_hdr_t *event_hdr) return (odp_packet_t)(uintptr_t)event_hdr; } +static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr) +{ + return pkt_hdr->seg_len; +} + static inline odp_packet_hdr_t *packet_last_seg(odp_packet_hdr_t *hdr) { while (hdr->seg_next != NULL) @@ -400,11 +406,6 @@ static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len) int _odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, odp_packet_t pkt[], int max_num); -/* Perform packet parse up to a given protocol layer */ -int _odp_packet_parse_layer(odp_packet_hdr_t *pkt_hdr, - odp_proto_layer_t layer, - odp_proto_chksums_t chksums); - /* Reset parser metadata for a new parse */ static inline void packet_parse_reset(odp_packet_hdr_t *pkt_hdr, int all) { @@ -454,12 +455,6 @@ static inline void packet_set_ts(odp_packet_hdr_t *pkt_hdr, odp_time_t *ts) } } -int _odp_packet_parse_common(packet_parser_t *pkt_hdr, const uint8_t *ptr, - uint32_t pkt_len, uint32_t seg_len, int layer, - odp_proto_chksums_t chksums); - -int _odp_cls_parse(odp_packet_hdr_t *pkt_hdr, const uint8_t *parseptr); - int _odp_packet_set_data(odp_packet_t pkt, uint32_t offset, uint8_t c, uint32_t len); @@ -471,6 +466,9 @@ int _odp_packet_tcp_chksum_insert(odp_packet_t pkt); int _odp_packet_udp_chksum_insert(odp_packet_t pkt); int _odp_packet_sctp_chksum_insert(odp_packet_t pkt); +int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, + odp_proto_chksums_t chksums, uint64_t l4_part_sum); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index 3e9f94100..aed6de412 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -78,6 +78,7 @@ struct pktio_entry { /* These two locks together lock the whole pktio device */ odp_ticketlock_t rxl; /**< RX ticketlock */ odp_ticketlock_t txl; /**< TX ticketlock */ + odp_proto_layer_t parse_layer; uint16_t pktin_frame_offset; struct { @@ -93,6 +94,8 @@ struct pktio_entry { uint8_t tx_ts : 1; /* Tx completion events */ uint8_t tx_compl : 1; + /* Packet aging */ + uint8_t tx_aging : 1; }; }; } enabled; @@ -144,8 +147,8 @@ struct pktio_entry { /* Storage for queue handles * Multi-queue support is pktio driver specific */ - unsigned num_in_queue; - unsigned num_out_queue; + uint32_t num_in_queue; + uint32_t num_out_queue; struct { odp_queue_t queue; @@ -227,8 +230,8 @@ typedef struct pktio_if_ops { int num); int (*recv_tmo)(pktio_entry_t *entry, int index, odp_packet_t packets[], int num, uint64_t wait_usecs); - int (*recv_mq_tmo)(pktio_entry_t *entry[], int index[], int num_q, - odp_packet_t packets[], int num, unsigned *from, + int (*recv_mq_tmo)(pktio_entry_t *entry[], int index[], uint32_t num_q, + odp_packet_t packets[], int num, uint32_t *from, uint64_t wait_usecs); int (*fd_set)(pktio_entry_t *entry, int index, fd_set *readfds); int (*send)(pktio_entry_t *entry, int index, @@ -291,6 +294,11 @@ static inline int _odp_pktio_tx_compl_enabled(const pktio_entry_t *entry) return entry->s.enabled.tx_compl; } +static inline int _odp_pktio_tx_aging_enabled(pktio_entry_t *entry) +{ + return entry->s.enabled.tx_aging; +} + static inline void _odp_pktio_tx_ts_set(pktio_entry_t *entry) { odp_time_t ts_val = odp_time_global(); @@ -326,7 +334,7 @@ extern const pktio_if_ops_t * const _odp_pktio_if_ops[]; * @return <0 on failure */ int _odp_sock_recv_mq_tmo_try_int_driven(const struct odp_pktin_queue_t queues[], - unsigned int num_q, unsigned int *from, + uint32_t num_q, uint32_t *from, odp_packet_t packets[], int num, uint64_t usecs, int *trial_successful); diff --git a/platform/linux-generic/include/odp_parse_internal.h b/platform/linux-generic/include/odp_parse_internal.h new file mode 100644 index 000000000..8aa5e118b --- /dev/null +++ b/platform/linux-generic/include/odp_parse_internal.h @@ -0,0 +1,112 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2019-2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PARSE_INTERNAL_H_ +#define ODP_PARSE_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp_packet_internal.h> +#include <odp_macros_internal.h> +#include <protocols/eth.h> +#include <protocols/ip.h> +#include <protocols/sctp.h> +#include <protocols/tcp.h> +#include <protocols/udp.h> +#include <odp/api/plat/packet_inline_types.h> +#include <odp/api/hints.h> +#include <odp/api/packet_io.h> +#include <odp/api/packet_types.h> +#include <stdint.h> + +/* + * In the worst case we look at the Ethernet header, 8 bytes of LLC/SNAP + * header and two VLAN tags in the same packet. + */ +#define PARSE_ETH_BYTES (sizeof(_odp_ethhdr_t) + 8 + 2 * sizeof(_odp_vlanhdr_t)) +#define PARSE_IPV4_BYTES (0xfU * 4) /* max IPv4 header length with options */ +/* + * Peek 2 bytes beyond IPv6 base header without length check if there are + * extension headers. + */ +#define PARSE_IPV6_BYTES (sizeof(_odp_ipv6hdr_t) + 2) +#define PARSE_TCP_BYTES (sizeof(_odp_tcphdr_t)) +/* + * In the worst case we look at the UDP header and 4 bytes of the UDP + * payload (the non-ESP marker to distinguish IKE packets from ESP packets). + */ +#define PARSE_UDP_BYTES (sizeof(_odp_udphdr_t) + 4) +#define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t)) + +/* _odp_packet_parse_common_l3_l4() requires up to this many bytes. */ +#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \ + MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES)) + +/* _odp_packet_parse_common() requires up to this many bytes. */ +#define PARSE_BYTES (PARSE_ETH_BYTES + PARSE_L3_L4_BYTES) + +uint16_t _odp_parse_eth(packet_parser_t *prs, const uint8_t **parseptr, + uint32_t *offset, uint32_t frame_len); + +/* + * Parse common L3 and L4 packet headers up to given layer + * + * See _odp_packet_parse_common(). Requires up to PARSE_L3_L4_BYTES bytes of + * contiguous packet data. + */ +int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, + const uint8_t *parseptr, uint32_t offset, + uint32_t frame_len, uint32_t seg_len, + int layer, uint16_t ethtype, + odp_proto_chksums_t chksums, + uint64_t *l4_part_sum, + odp_pktin_config_opt_t opt); + +/** + * Parse common packet headers up to given layer + * + * Requires up to PARSE_BYTES bytes of contiguous packet data. Also parse + * metadata must be already initialized. + * + * Returns 0 on success, 1 on packet errors, and -1 if the packet should be + * dropped. + */ +static inline int _odp_packet_parse_common(packet_parser_t *prs, + const uint8_t *ptr, + uint32_t frame_len, uint32_t seg_len, + int layer, + odp_proto_chksums_t chksums, + uint64_t *l4_part_sum, + odp_pktin_config_opt_t opt) +{ + uint32_t offset; + uint16_t ethtype; + const uint8_t *parseptr; + + parseptr = ptr; + offset = 0; + + if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE)) + return 0; + + /* Assume valid L2 header, no CRC/FCS check in SW */ + prs->l2_offset = offset; + + ethtype = _odp_parse_eth(prs, &parseptr, &offset, frame_len); + + return _odp_packet_parse_common_l3_l4(prs, parseptr, offset, frame_len, + seg_len, layer, ethtype, chksums, + l4_part_sum, opt); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index ba4ad53e8..cec9c8bb8 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -89,8 +89,6 @@ extern const schedule_fn_t *_odp_sched_fn; /* Interface for the scheduler */ int _odp_sched_cb_pktin_poll(int pktio_index, int pktin_index, _odp_event_hdr_t *hdr_tbl[], int num); -/* evts must have at least QUEUE_MULTI_MAX elements */ -int _odp_sched_cb_pktin_poll_one(int pktio_index, int rx_queue, odp_event_t evts[]); void _odp_sched_cb_pktio_stop_finalize(int pktio_index); /* API functions */ diff --git a/platform/linux-generic/include/odp_types_internal.h b/platform/linux-generic/include/odp_types_internal.h new file mode 100644 index 000000000..a97ac9cd4 --- /dev/null +++ b/platform/linux-generic/include/odp_types_internal.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_TYPES_INTERNAL_H_ +#define ODP_TYPES_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __SIZEOF_INT128__ + +__extension__ typedef unsigned __int128 _odp_u128_t; + +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index 3b50232d6..2fdecbc51 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -150,7 +150,7 @@ void odp_cls_pmr_param_init(odp_pmr_param_t *param) int odp_cls_capability(odp_cls_capability_t *capability) { - unsigned int count = 0; + uint32_t count = 0; memset(capability, 0, sizeof(odp_cls_capability_t)); @@ -161,6 +161,7 @@ int odp_cls_capability(odp_cls_capability_t *capability) capability->max_pmr_terms = CLS_PMR_MAX_ENTRY; capability->available_pmr_terms = count; capability->max_cos = CLS_COS_MAX_ENTRY; + capability->max_cos_stats = capability->max_cos; capability->pmr_range_supported = false; capability->supported_terms.all_bits = 0; capability->supported_terms.bit.len = 1; @@ -186,6 +187,8 @@ int odp_cls_capability(odp_cls_capability_t *capability) capability->threshold_bp.all_bits = 0; capability->max_hash_queues = CLS_COS_QUEUE_MAX; capability->max_mark = MAX_MARK; + capability->stats.cos.counter.discards = 1; + capability->stats.cos.counter.packets = 1; capability->stats.queue.counter.discards = 1; capability->stats.queue.counter.packets = 1; @@ -220,21 +223,29 @@ static inline void _cls_queue_unwind(uint32_t tbl_index, uint32_t j) odp_queue_destroy(queue_grp_tbl->s.queue[tbl_index + --j]); } -odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param) +odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param_in) { uint32_t i, j; odp_queue_t queue; odp_cls_drop_t drop_policy; cos_t *cos; uint32_t tbl_index; + odp_cls_cos_param_t param = *param_in; + + if (param.action == ODP_COS_ACTION_DROP) { + param.num_queue = 1; + param.queue = ODP_QUEUE_INVALID; + param.pool = ODP_POOL_INVALID; + param.vector.enable = false; + } /* num_queue should not be zero */ - if (param->num_queue > CLS_COS_QUEUE_MAX || param->num_queue < 1) + if (param.num_queue > CLS_COS_QUEUE_MAX || param.num_queue < 1) return ODP_COS_INVALID; /* Validate packet vector parameters */ - if (param->vector.enable) { - odp_pool_t pool = param->vector.pool; + if (param.vector.enable) { + odp_pool_t pool = param.vector.pool; odp_pool_info_t pool_info; if (pool == ODP_POOL_INVALID || odp_pool_info(pool, &pool_info)) { @@ -245,17 +256,17 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param) ODP_ERR("wrong pool type\n"); return ODP_COS_INVALID; } - if (param->vector.max_size == 0) { + if (param.vector.max_size == 0) { ODP_ERR("vector.max_size is zero\n"); return ODP_COS_INVALID; } - if (param->vector.max_size > pool_info.params.vector.max_size) { + if (param.vector.max_size > pool_info.params.vector.max_size) { ODP_ERR("vector.max_size larger than pool max vector size\n"); return ODP_COS_INVALID; } } - drop_policy = param->drop_policy; + drop_policy = param.drop_policy; for (i = 0; i < CLS_COS_MAX_ENTRY; i++) { cos = &cos_tbl->cos_entry[i]; @@ -274,16 +285,16 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param) cos->s.linked_cos[j] = NULL; } - cos->s.num_queue = param->num_queue; + cos->s.num_queue = param.num_queue; - if (param->num_queue > 1) { + if (param.num_queue > 1) { odp_queue_param_init(&cos->s.queue_param); cos->s.queue_group = true; cos->s.queue = ODP_QUEUE_INVALID; _odp_cls_update_hash_proto(cos, - param->hash_proto); + param.hash_proto); tbl_index = i * CLS_COS_QUEUE_MAX; - for (j = 0; j < param->num_queue; j++) { + for (j = 0; j < param.num_queue; j++) { queue = odp_queue_create(NULL, &cos->s.queue_param); if (queue == ODP_QUEUE_INVALID) { /* unwind the queues */ @@ -296,21 +307,27 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param) } } else { - cos->s.queue = param->queue; + cos->s.queue = param.queue; } + + odp_atomic_init_u64(&cos->s.stats.discards, 0); + odp_atomic_init_u64(&cos->s.stats.packets, 0); + /* Initialize statistics counters */ for (j = 0; j < cos->s.num_queue; j++) { - odp_atomic_init_u64(&cos->s.stats[j].discards, 0); - odp_atomic_init_u64(&cos->s.stats[j].packets, 0); + odp_atomic_init_u64(&cos->s.queue_stats[j].discards, 0); + odp_atomic_init_u64(&cos->s.queue_stats[j].packets, 0); } - cos->s.pool = param->pool; + cos->s.action = param.action; + cos->s.pool = param.pool; cos->s.headroom = 0; cos->s.valid = 1; cos->s.drop_policy = drop_policy; odp_atomic_init_u32(&cos->s.num_rule, 0); cos->s.index = i; - cos->s.vector = param->vector; + cos->s.vector = param.vector; + cos->s.stats_enable = param.stats_enable; UNLOCK(&cos->s.lock); return _odp_cos_from_ndx(i); } @@ -1461,6 +1478,9 @@ static cos_t *match_pmr_cos(cos_t *cos, const uint8_t *pkt_addr, pmr_t *pmr, /* PMR matched */ pmr_debug_print(pmr, cos); + if (cos->s.stats_enable) + odp_atomic_inc_u64(&cos->s.stats.packets); + hdr->p.input_flags.cls_mark = 0; if (pmr->s.mark) { hdr->p.input_flags.cls_mark = 1; @@ -1537,8 +1557,11 @@ static inline cos_t *cls_select_cos(pktio_entry_t *entry, default_cos = cls->default_cos; /* Return error cos for error packet */ - if (pkt_hdr->p.flags.all.error) - return cls->error_cos; + if (pkt_hdr->p.flags.all.error) { + cos = cls->error_cos; + goto done; + } + /* Calls all the PMRs attached at the PKTIO level*/ for (i = 0; i < odp_atomic_load_u32(&default_cos->s.num_rule); i++) { pmr = default_cos->s.pmr[i]; @@ -1551,11 +1574,17 @@ static inline cos_t *cls_select_cos(pktio_entry_t *entry, cos = match_qos_cos(entry, pkt_addr, pkt_hdr); if (cos) { ODP_DBG_RAW(CLS_DBG, " QoS matched -> cos: %s(%u)\n", cos->s.name, cos->s.index); - return cos; + goto done; } ODP_DBG_RAW(CLS_DBG, " No match -> default cos\n"); - return cls->default_cos; + cos = cls->default_cos; + +done: + if (cos && cos->s.stats_enable) + odp_atomic_inc_u64(&cos->s.stats.packets); + + return cos; } static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr, @@ -1575,12 +1604,12 @@ static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr, * @retval 0 on success * @retval -EFAULT Bug * @retval -EINVAL Config error + * @retval -ENOENT Drop action * * @note *base is not released */ int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base, - uint16_t pkt_len, uint32_t seg_len, odp_pool_t *pool, - odp_packet_hdr_t *pkt_hdr, odp_bool_t parse) + odp_pool_t *pool, odp_packet_hdr_t *pkt_hdr) { cos_t *cos; uint32_t tbl_index; @@ -1588,24 +1617,19 @@ int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base, ODP_DBG_LVL(CLS_DBG, "Classify packet from %s\n", entry->s.full_name); - if (parse) { - packet_parse_reset(pkt_hdr, 1); - packet_set_len(pkt_hdr, pkt_len); - - _odp_packet_parse_common(&pkt_hdr->p, base, pkt_len, seg_len, - ODP_PROTO_LAYER_ALL, - entry->s.in_chksums); - } cos = cls_select_cos(entry, base, pkt_hdr); if (cos == NULL) return -EINVAL; + if (cos->s.action == ODP_COS_ACTION_DROP) + return -ENOENT; + if (cos->s.queue == ODP_QUEUE_INVALID && cos->s.num_queue == 1) - return -EFAULT; + goto error; if (cos->s.pool == ODP_POOL_INVALID) - return -EFAULT; + goto error; *pool = cos->s.pool; pkt_hdr->p.input_flags.dst_queue = 1; @@ -1623,6 +1647,10 @@ int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base, cos->s.num_queue); pkt_hdr->dst_queue = queue_grp_tbl->s.queue[tbl_index]; return 0; + +error: + odp_atomic_inc_u64(&cos->s.stats.discards); + return -EFAULT; } static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr, @@ -1780,6 +1808,27 @@ uint64_t odp_pmr_to_u64(odp_pmr_t hdl) return _odp_pri(hdl); } +int odp_cls_cos_stats(odp_cos_t hdl, odp_cls_cos_stats_t *stats) +{ + cos_t *cos = get_cos_entry(hdl); + + if (odp_unlikely(cos == NULL)) { + ODP_ERR("Invalid odp_cos_t handle\n"); + return -1; + } + + if (odp_unlikely(stats == NULL)) { + ODP_ERR("Output structure NULL\n"); + return -1; + } + + memset(stats, 0, sizeof(*stats)); + stats->discards = odp_atomic_load_u64(&cos->s.stats.discards); + stats->packets = odp_atomic_load_u64(&cos->s.stats.packets); + + return 0; +} + int odp_cls_queue_stats(odp_cos_t hdl, odp_queue_t queue, odp_cls_queue_stats_t *stats) { @@ -1803,8 +1852,8 @@ int odp_cls_queue_stats(odp_cos_t hdl, odp_queue_t queue, } memset(stats, 0, sizeof(odp_cls_queue_stats_t)); - stats->discards = odp_atomic_load_u64(&cos->s.stats[queue_idx].discards); - stats->packets = odp_atomic_load_u64(&cos->s.stats[queue_idx].packets); + stats->discards = odp_atomic_load_u64(&cos->s.queue_stats[queue_idx].discards); + stats->packets = odp_atomic_load_u64(&cos->s.queue_stats[queue_idx].packets); return 0; } diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index bc14bad9a..f276d4659 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -269,9 +269,7 @@ int odp_crypto_session_destroy(odp_crypto_session_t session) return 0; } -/* - * Shim function around packet operation, can be used by other implementations. - */ +#if ODP_DEPRECATED_API int odp_crypto_operation(odp_crypto_op_param_t *param, odp_bool_t *posted, @@ -326,6 +324,7 @@ odp_crypto_operation(odp_crypto_op_param_t *param, return 0; } +#endif int _odp_crypto_init_global(void) @@ -402,6 +401,7 @@ int _odp_crypto_term_local(void) return 0; } +#if ODP_DEPRECATED_API odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev) { /* This check not mandated by the API specification */ @@ -438,6 +438,7 @@ uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl) { return _odp_pri(hdl); } +#endif void odp_crypto_session_param_init(odp_crypto_session_param_t *param) { diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 13101fbca..6ff0ac041 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -31,6 +31,7 @@ #include <openssl/hmac.h> #include <openssl/cmac.h> #include <openssl/evp.h> +#include <openssl/opensslv.h> #if (OPENSSL_VERSION_NUMBER >= 0x10100000L) && !defined(OPENSSL_NO_POLY1305) #define _ODP_HAVE_CHACHA20_POLY1305 1 @@ -38,6 +39,11 @@ #define _ODP_HAVE_CHACHA20_POLY1305 0 #endif +/* Ignore warnings about APIs deprecated in OpenSSL 3.0 */ +#if OPENSSL_VERSION_NUMBER >= 0x30000000L +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + #define MAX_SESSIONS 4000 #define AES_BLOCK_SIZE 16 #define AES_KEY_LENGTH 16 @@ -2495,6 +2501,7 @@ int odp_crypto_session_destroy(odp_crypto_session_t session) /* * Shim function around packet operation, can be used by other implementations. */ +#if ODP_DEPRECATED_API int odp_crypto_operation(odp_crypto_op_param_t *param, odp_bool_t *posted, @@ -2549,6 +2556,7 @@ odp_crypto_operation(odp_crypto_op_param_t *param, return 0; } +#endif #if OPENSSL_VERSION_NUMBER < 0x10100000L static void ODP_UNUSED openssl_thread_id(CRYPTO_THREADID ODP_UNUSED *id) @@ -2705,6 +2713,7 @@ int _odp_crypto_term_local(void) return 0; } +#if ODP_DEPRECATED_API odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev) { /* This check not mandated by the API specification */ @@ -2741,6 +2750,7 @@ uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl) { return _odp_pri(hdl); } +#endif /* ODP_DEPRECATED_API */ void odp_crypto_session_param_init(odp_crypto_session_param_t *param) { diff --git a/platform/linux-generic/odp_dma.c b/platform/linux-generic/odp_dma.c index 87e3b5b73..4c129c616 100644 --- a/platform/linux-generic/odp_dma.c +++ b/platform/linux-generic/odp_dma.c @@ -10,6 +10,7 @@ #include <odp/api/align.h> #include <odp/api/buffer.h> #include <odp/api/stash.h> +#include <odp/api/packet.h> #include <odp/api/pool.h> #include <odp/api/queue.h> diff --git a/platform/linux-generic/odp_event.c b/platform/linux-generic/odp_event.c index 95be9dec2..90c3e3a3c 100644 --- a/platform/linux-generic/odp_event.c +++ b/platform/linux-generic/odp_event.c @@ -24,6 +24,7 @@ #include <odp/api/plat/event_inlines.h> #include <odp/api/plat/packet_inlines.h> #include <odp/api/plat/packet_vector_inlines.h> +#include <odp/api/plat/timer_inlines.h> odp_event_subtype_t odp_event_subtype(odp_event_t event) { @@ -70,9 +71,11 @@ void odp_event_free(odp_event_t event) case ODP_EVENT_TIMEOUT: odp_timeout_free(odp_timeout_from_event(event)); break; +#if ODP_DEPRECATED_API case ODP_EVENT_CRYPTO_COMPL: odp_crypto_compl_free(odp_crypto_compl_from_event(event)); break; +#endif case ODP_EVENT_IPSEC_STATUS: _odp_ipsec_status_free(_odp_ipsec_status_from_event(event)); break; @@ -120,8 +123,10 @@ int odp_event_is_valid(odp_event_t event) /* Fall through */ case ODP_EVENT_TIMEOUT: /* Fall through */ +#if ODP_DEPRECATED_API case ODP_EVENT_CRYPTO_COMPL: /* Fall through */ +#endif case ODP_EVENT_IPSEC_STATUS: /* Fall through */ case ODP_EVENT_PACKET_VECTOR: diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 4dec8f440..29f0e4bcd 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -1803,6 +1803,7 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, state.out_tunnel.ip_tos = 0; state.out_tunnel.ip_df = 0; state.out_tunnel.ip_flabel = 0; + state.ip_next_hdr = _ODP_IPPROTO_NO_NEXT; rc = 0; } else { rc = -1; @@ -2269,12 +2270,6 @@ int odp_ipsec_out_inline(const odp_packet_t pkt_in[], int num_in, ptr) < 0) status.error.alg = 1; - packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); - result = ipsec_pkt_result(pkt); - memset(result, 0, sizeof(*result)); - result->sa = ipsec_sa->ipsec_sa_hdl; - result->status = status; - if (!status.error.all) { odp_pktout_queue_t pkqueue; diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index c47e4912e..4e4057ede 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -863,7 +863,10 @@ ipsec_sa_t *_odp_ipsec_sa_lookup(const ipsec_sa_lookup_t *lookup) lookup->proto == ipsec_sa->proto && lookup->spi == ipsec_sa->spi && lookup->ver == ipsec_sa->in.lookup_ver && - !memcmp(lookup->dst_addr, &ipsec_sa->in.lookup_dst_ipv4, + !memcmp(lookup->dst_addr, + lookup->ver == ODP_IPSEC_IPV4 ? + (void *)&ipsec_sa->in.lookup_dst_ipv4 : + (void *)&ipsec_sa->in.lookup_dst_ipv6, lookup->ver == ODP_IPSEC_IPV4 ? _ODP_IPV4ADDR_LEN : _ODP_IPV6ADDR_LEN)) { diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index b91f8ca70..b0d0fc03c 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -16,6 +16,7 @@ #include <odp/api/proto_stats.h> #include <odp/api/timer.h> +#include <odp_parse_internal.h> #include <odp_chksum_internal.h> #include <odp_debug_internal.h> #include <odp_errno_define.h> @@ -131,11 +132,6 @@ static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, return pkt_hdr->seg_len; } -static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr) -{ - return pkt_hdr->seg_len; -} - static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr) { odp_packet_hdr_t *last_seg = packet_last_seg(pkt_hdr); @@ -1771,500 +1767,6 @@ static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr, return sum; } -/* - * In the worst case we look at the Ethernet header, 8 bytes of LLC/SNAP - * header and two VLAN tags in the same packet. - */ -#define PARSE_ETH_BYTES (sizeof(_odp_ethhdr_t) + 8 + 2 * sizeof(_odp_vlanhdr_t)) -/** Parser helper function for Ethernet packets - * - * Requires up to PARSE_ETH_BYTES bytes of contiguous packet data. - */ -static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, - uint32_t *offset, uint32_t frame_len) -{ - uint16_t ethtype; - const _odp_ethhdr_t *eth; - uint16_t macaddr0, macaddr2, macaddr4; - const _odp_vlanhdr_t *vlan; - _odp_packet_input_flags_t input_flags; - - input_flags.all = 0; - input_flags.l2 = 1; - input_flags.eth = 1; - - eth = (const _odp_ethhdr_t *)*parseptr; - - /* Detect jumbo frames */ - if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX)) - input_flags.jumbo = 1; - - /* Handle Ethernet broadcast/multicast addresses */ - macaddr0 = odp_be_to_cpu_16(*((const uint16_t *)(const void *)eth)); - if (odp_unlikely((macaddr0 & 0x0100) == 0x0100)) - input_flags.eth_mcast = 1; - - if (odp_unlikely(macaddr0 == 0xffff)) { - macaddr2 = - odp_be_to_cpu_16(*((const uint16_t *) - (const void *)eth + 1)); - macaddr4 = - odp_be_to_cpu_16(*((const uint16_t *) - (const void *)eth + 2)); - - if ((macaddr2 == 0xffff) && (macaddr4 == 0xffff)) - input_flags.eth_bcast = 1; - } - - /* Get Ethertype */ - ethtype = odp_be_to_cpu_16(eth->type); - *offset += sizeof(*eth); - *parseptr += sizeof(*eth); - - /* Check for SNAP vs. DIX */ - if (odp_unlikely(ethtype < _ODP_ETH_LEN_MAX)) { - input_flags.snap = 1; - if (ethtype > frame_len - *offset) { - prs->flags.snap_len_err = 1; - ethtype = 0; - goto error; - } - ethtype = odp_be_to_cpu_16(*((const uint16_t *)(uintptr_t) - (*parseptr + 6))); - *offset += 8; - *parseptr += 8; - } - - /* Parse the VLAN header(s), if present */ - if (odp_unlikely(ethtype == _ODP_ETHTYPE_VLAN_OUTER)) { - input_flags.vlan_qinq = 1; - input_flags.vlan = 1; - - vlan = (const _odp_vlanhdr_t *)*parseptr; - ethtype = odp_be_to_cpu_16(vlan->type); - *offset += sizeof(_odp_vlanhdr_t); - *parseptr += sizeof(_odp_vlanhdr_t); - } - - if (ethtype == _ODP_ETHTYPE_VLAN) { - input_flags.vlan = 1; - vlan = (const _odp_vlanhdr_t *)*parseptr; - ethtype = odp_be_to_cpu_16(vlan->type); - *offset += sizeof(_odp_vlanhdr_t); - *parseptr += sizeof(_odp_vlanhdr_t); - } - - /* - * The packet was too short for what we parsed. We just give up - * entirely without trying to parse what fits in the packet. - */ - if (odp_unlikely(*offset > frame_len)) { - input_flags.all = 0; - input_flags.l2 = 1; - ethtype = 0; - } - -error: - prs->input_flags.all |= input_flags.all; - - return ethtype; -} - -#define PARSE_IPV4_BYTES (0xfU * 4) /* max IPv4 header length with options */ -/** - * Parser helper function for IPv4 - * - * Requires up to PARSE_IPV4_BYTES bytes of contiguous packet data. - */ -static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, - uint32_t *offset, uint32_t frame_len, - odp_proto_chksums_t chksums, - uint64_t *l4_part_sum) -{ - const _odp_ipv4hdr_t *ipv4 = (const _odp_ipv4hdr_t *)*parseptr; - uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr); - uint32_t l3_len = odp_be_to_cpu_16(ipv4->tot_len); - uint16_t frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); - uint8_t ver = _ODP_IPV4HDR_VER(ipv4->ver_ihl); - uint8_t ihl = _ODP_IPV4HDR_IHL(ipv4->ver_ihl); - - if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN || - ver != 4 || - sizeof(*ipv4) > frame_len - *offset || - (l3_len > frame_len - *offset))) { - prs->flags.ip_err = 1; - return 0; - } - - if (chksums.chksum.ipv4) { - prs->input_flags.l3_chksum_done = 1; - if (chksum_finalize(chksum_partial(ipv4, ihl * 4, 0)) != 0xffff) { - prs->flags.ip_err = 1; - prs->flags.l3_chksum_err = 1; - return 0; - } - } - - *offset += ihl * 4; - *parseptr += ihl * 4; - - if (chksums.chksum.udp || chksums.chksum.tcp) - *l4_part_sum = chksum_partial((const uint8_t *)&ipv4->src_addr, - 2 * _ODP_IPV4ADDR_LEN, 0); - - if (odp_unlikely(ihl > _ODP_IPV4HDR_IHL_MIN)) - prs->input_flags.ipopt = 1; - - /* A packet is a fragment if: - * "more fragments" flag is set (all fragments except the last) - * OR - * "fragment offset" field is nonzero (all fragments except the first) - */ - if (odp_unlikely(_ODP_IPV4HDR_IS_FRAGMENT(frag_offset))) - prs->input_flags.ipfrag = 1; - - /* Handle IPv4 broadcast / multicast */ - if (odp_unlikely(dstaddr == 0xffffffff)) - prs->input_flags.ip_bcast = 1; - - if (odp_unlikely((dstaddr >> 28) == 0xe)) - prs->input_flags.ip_mcast = 1; - - return ipv4->proto; -} - -/* - * Peeks 2 bytes beyond IPv6 base header without length check if there - * are extension headers. - */ -#define PARSE_IPV6_BYTES (sizeof(_odp_ipv6hdr_t) + 2) -/** - * Parser helper function for IPv6 - * - * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data. - */ -static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, - uint32_t *offset, uint32_t frame_len, - uint32_t seg_len, - odp_proto_chksums_t chksums, - uint64_t *l4_part_sum) -{ - const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr; - const _odp_ipv6hdr_ext_t *ipv6ext; - uint32_t dstaddr0 = odp_be_to_cpu_32(ipv6->dst_addr.u8[0]); - uint32_t l3_len = odp_be_to_cpu_16(ipv6->payload_len) + - _ODP_IPV6HDR_LEN; - - /* Basic sanity checks on IPv6 header */ - if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || - sizeof(*ipv6) > frame_len - *offset || - l3_len > frame_len - *offset)) { - prs->flags.ip_err = 1; - return 0; - } - - /* IPv6 broadcast / multicast flags */ - prs->input_flags.ip_mcast = (dstaddr0 & 0xff000000) == 0xff000000; - prs->input_flags.ip_bcast = 0; - - /* Skip past IPv6 header */ - *offset += sizeof(_odp_ipv6hdr_t); - *parseptr += sizeof(_odp_ipv6hdr_t); - - if (chksums.chksum.udp || chksums.chksum.tcp) - *l4_part_sum = chksum_partial((const uint8_t *)&ipv6->src_addr, - 2 * _ODP_IPV6ADDR_LEN, 0); - - /* Skip past any IPv6 extension headers */ - if (ipv6->next_hdr == _ODP_IPPROTO_HOPOPTS || - ipv6->next_hdr == _ODP_IPPROTO_ROUTE) { - prs->input_flags.ipopt = 1; - - do { - ipv6ext = (const _odp_ipv6hdr_ext_t *)*parseptr; - uint16_t extlen = 8 + ipv6ext->ext_len * 8; - - *offset += extlen; - *parseptr += extlen; - } while ((ipv6ext->next_hdr == _ODP_IPPROTO_HOPOPTS || - ipv6ext->next_hdr == _ODP_IPPROTO_ROUTE) && - *offset < seg_len); - - if (*offset >= prs->l3_offset + - odp_be_to_cpu_16(ipv6->payload_len)) { - prs->flags.ip_err = 1; - return 0; - } - - if (ipv6ext->next_hdr == _ODP_IPPROTO_FRAG) - prs->input_flags.ipfrag = 1; - - return ipv6ext->next_hdr; - } - - if (odp_unlikely(ipv6->next_hdr == _ODP_IPPROTO_FRAG)) { - prs->input_flags.ipopt = 1; - prs->input_flags.ipfrag = 1; - } - - return ipv6->next_hdr; -} - -#define PARSE_TCP_BYTES (sizeof(_odp_tcphdr_t)) -/** - * Parser helper function for TCP - * - * Requires PARSE_TCP_BYTES bytes of contiguous packet data. - */ -static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, - uint16_t tcp_len, - odp_proto_chksums_t chksums, - uint64_t *l4_part_sum) -{ - const _odp_tcphdr_t *tcp = (const _odp_tcphdr_t *)*parseptr; - uint32_t len = tcp->hl * 4; - - if (odp_unlikely(tcp->hl < sizeof(_odp_tcphdr_t) / sizeof(uint32_t))) - prs->flags.tcp_err = 1; - - if (chksums.chksum.tcp && - !prs->input_flags.ipfrag) { - *l4_part_sum += odp_cpu_to_be_16(tcp_len); -#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN - *l4_part_sum += _ODP_IPPROTO_TCP; -#else - *l4_part_sum += _ODP_IPPROTO_TCP << 8; -#endif - } - - *parseptr += len; -} - -/* - * In the worst case we look at the UDP header and 4 bytes of the UDP - * payload (the non-ESP marker to distinguish IKE packets from ESP packets). - */ -#define PARSE_UDP_BYTES (sizeof(_odp_udphdr_t) + 4) -/** - * Parser helper function for UDP - * - * Requires PARSE_UDP_BYTES bytes of contiguous packet data. - */ -static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, - odp_proto_chksums_t chksums, - uint64_t *l4_part_sum) -{ - const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr; - uint32_t udplen = odp_be_to_cpu_16(udp->length); - uint16_t ipsec_port = odp_cpu_to_be_16(_ODP_UDP_IPSEC_PORT); - - if (odp_unlikely(udplen < sizeof(_odp_udphdr_t))) { - prs->flags.udp_err = 1; - return; - } - - if (chksums.chksum.udp && - !prs->input_flags.ipfrag) { - if (udp->chksum == 0) { - prs->input_flags.l4_chksum_done = 1; - prs->flags.l4_chksum_err = - (prs->input_flags.ipv4 != 1); - } else { - *l4_part_sum += udp->length; -#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN - *l4_part_sum += _ODP_IPPROTO_UDP; -#else - *l4_part_sum += _ODP_IPPROTO_UDP << 8; -#endif - } - prs->input_flags.udp_chksum_zero = (udp->chksum == 0); - } - - if (odp_unlikely(ipsec_port == udp->dst_port && udplen > 4)) { - uint32_t val; - - memcpy(&val, udp + 1, 4); - if (val != 0) { - prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_udp = 1; - } - } - - *parseptr += sizeof(_odp_udphdr_t); -} - -#define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t)) -/** - * Parser helper function for SCTP - * - * Requires PARSE_SCTP_BYTES bytes of contiguous packet data. - */ -static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, - uint16_t sctp_len, - odp_proto_chksums_t chksums, - uint64_t *l4_part_sum) -{ - if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) { - prs->flags.sctp_err = 1; - return; - } - - if (chksums.chksum.sctp && - !prs->input_flags.ipfrag) { - const _odp_sctphdr_t *sctp = - (const _odp_sctphdr_t *)*parseptr; - uint32_t crc = ~0; - uint32_t zero = 0; - - crc = odp_hash_crc32c(sctp, sizeof(*sctp) - 4, crc); - crc = odp_hash_crc32c(&zero, 4, crc); - *l4_part_sum = crc; - } - - *parseptr += sizeof(_odp_sctphdr_t); -} - -#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c))) -#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \ - MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES)) -/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */ -static inline -int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, - uint32_t offset, - uint32_t frame_len, uint32_t seg_len, - int layer, uint16_t ethtype, - odp_proto_chksums_t chksums, - uint64_t *l4_part_sum) -{ - uint8_t ip_proto; - - prs->l3_offset = offset; - - if (odp_unlikely(layer <= ODP_PROTO_LAYER_L2)) - return prs->flags.all.error != 0; - - /* Set l3 flag only for known ethtypes */ - prs->input_flags.l3 = 1; - - /* Parse Layer 3 headers */ - switch (ethtype) { - case _ODP_ETHTYPE_IPV4: - prs->input_flags.ipv4 = 1; - ip_proto = parse_ipv4(prs, &parseptr, &offset, frame_len, - chksums, l4_part_sum); - prs->l4_offset = offset; - break; - - case _ODP_ETHTYPE_IPV6: - prs->input_flags.ipv6 = 1; - ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len, - seg_len, chksums, l4_part_sum); - prs->l4_offset = offset; - break; - - case _ODP_ETHTYPE_ARP: - prs->input_flags.arp = 1; - ip_proto = 255; /* Reserved invalid by IANA */ - break; - - default: - prs->input_flags.l3 = 0; - ip_proto = 255; /* Reserved invalid by IANA */ - } - - if (layer == ODP_PROTO_LAYER_L3) - return prs->flags.all.error != 0; - - /* Set l4 flag only for known ip_proto */ - prs->input_flags.l4 = 1; - - /* Parse Layer 4 headers */ - switch (ip_proto) { - case _ODP_IPPROTO_ICMPV4: - /* Fall through */ - - case _ODP_IPPROTO_ICMPV6: - prs->input_flags.icmp = 1; - break; - - case _ODP_IPPROTO_IPIP: - /* Do nothing */ - break; - - case _ODP_IPPROTO_TCP: - if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len)) - return -1; - prs->input_flags.tcp = 1; - parse_tcp(prs, &parseptr, frame_len - prs->l4_offset, chksums, - l4_part_sum); - break; - - case _ODP_IPPROTO_UDP: - if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len)) - return -1; - prs->input_flags.udp = 1; - parse_udp(prs, &parseptr, chksums, l4_part_sum); - break; - - case _ODP_IPPROTO_AH: - prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_ah = 1; - break; - - case _ODP_IPPROTO_ESP: - prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_esp = 1; - break; - - case _ODP_IPPROTO_SCTP: - prs->input_flags.sctp = 1; - parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, chksums, - l4_part_sum); - break; - - case _ODP_IPPROTO_NO_NEXT: - prs->input_flags.no_next_hdr = 1; - break; - - default: - prs->input_flags.l4 = 0; - break; - } - - return prs->flags.all.error != 0; -} - -/** - * Parse common packet headers up to given layer - * - * The function expects at least PACKET_PARSE_SEG_LEN bytes of data to be - * available from the ptr. Also parse metadata must be already initialized. - */ -int _odp_packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, - uint32_t frame_len, uint32_t seg_len, - int layer, odp_proto_chksums_t chksums) -{ - uint32_t offset; - uint16_t ethtype; - const uint8_t *parseptr; - uint64_t l4_part_sum; - - parseptr = ptr; - offset = 0; - - if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE)) - return 0; - - /* Assume valid L2 header, no CRC/FCS check in SW */ - prs->l2_offset = offset; - - ethtype = parse_eth(prs, &parseptr, &offset, frame_len); - - return packet_parse_common_l3_l4(prs, parseptr, offset, frame_len, - seg_len, layer, ethtype, chksums, - &l4_part_sum); -} - static inline int packet_ipv4_chksum(odp_packet_t pkt, uint32_t offset, _odp_ipv4hdr_t *ip, @@ -2442,9 +1944,8 @@ int _odp_packet_sctp_chksum_insert(odp_packet_t pkt) return odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum); } -static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, - odp_proto_chksums_t chksums, - uint64_t l4_part_sum) +int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, + odp_proto_chksums_t chksums, uint64_t l4_part_sum) { /* UDP chksum == 0 case is covered in parse_udp() */ if (chksums.chksum.udp && @@ -2518,42 +2019,6 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, return pkt_hdr->p.flags.all.error != 0; } -/** - * Simple packet parser - */ -int _odp_packet_parse_layer(odp_packet_hdr_t *pkt_hdr, - odp_proto_layer_t layer, - odp_proto_chksums_t chksums) -{ - uint32_t seg_len = packet_first_seg_len(pkt_hdr); - const uint8_t *base = packet_data(pkt_hdr); - uint32_t offset = 0; - uint16_t ethtype; - uint64_t l4_part_sum = 0; - int rc; - - if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE)) - return 0; - - /* Assume valid L2 header, no CRC/FCS check in SW */ - pkt_hdr->p.l2_offset = offset; - - ethtype = parse_eth(&pkt_hdr->p, &base, &offset, pkt_hdr->frame_len); - - rc = packet_parse_common_l3_l4(&pkt_hdr->p, base, offset, - pkt_hdr->frame_len, - seg_len, layer, ethtype, chksums, - &l4_part_sum); - - if (rc != 0) - return rc; - - if (layer >= ODP_PROTO_LAYER_L4) - return packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); - else - return 0; -} - int odp_packet_parse(odp_packet_t pkt, uint32_t offset, const odp_packet_parse_param_t *param) { @@ -2568,6 +2033,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, uint64_t l4_part_sum = 0; const uint32_t min_seglen = PARSE_ETH_BYTES + PARSE_L3_L4_BYTES; uint8_t buf[min_seglen]; + odp_pktin_config_opt_t opt; if (proto == ODP_PROTO_NONE || layer == ODP_PROTO_LAYER_NONE) return -1; @@ -2598,7 +2064,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, /* Assume valid L2 header, no CRC/FCS check in SW */ pkt_hdr->p.l2_offset = offset; - ethtype = parse_eth(&pkt_hdr->p, &data, &offset, packet_len); + ethtype = _odp_parse_eth(&pkt_hdr->p, &data, &offset, packet_len); } else if (proto == ODP_PROTO_IPV4) { ethtype = _ODP_ETHTYPE_IPV4; } else if (proto == ODP_PROTO_IPV6) { @@ -2607,17 +2073,18 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, ethtype = 0; /* Invalid */ } - ret = packet_parse_common_l3_l4(&pkt_hdr->p, data, offset, - packet_len, seg_len, - layer, ethtype, - param->chksums, - &l4_part_sum); + opt.all_bits = 0; + + ret = _odp_packet_parse_common_l3_l4(&pkt_hdr->p, data, offset, + packet_len, seg_len, layer, + ethtype, param->chksums, + &l4_part_sum, opt); if (ret) return -1; if (layer >= ODP_PROTO_LAYER_L4) { - ret = packet_l4_chksum(pkt_hdr, param->chksums, l4_part_sum); + ret = _odp_packet_l4_chksum(pkt_hdr, param->chksums, l4_part_sum); if (ret) return -1; } @@ -2879,14 +2346,17 @@ int odp_packet_payload_offset_set(odp_packet_t pkt, uint32_t offset) void odp_packet_aging_tmo_set(odp_packet_t pkt, uint64_t tmo_ns) { - (void)pkt; - (void)tmo_ns; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + pkt_hdr->p.flags.tx_aging = tmo_ns ? 1 : 0; + pkt_hdr->tx_aging_ns = tmo_ns; } uint64_t odp_packet_aging_tmo(odp_packet_t pkt) { - (void)pkt; - return 0; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + return pkt_hdr->p.flags.tx_aging ? pkt_hdr->tx_aging_ns : 0; } int odp_packet_tx_compl_request(odp_packet_t pkt, const odp_packet_tx_compl_opt_t *opt) diff --git a/platform/linux-generic/odp_packet_flags.c b/platform/linux-generic/odp_packet_flags.c index bb5c26792..ef542a2cd 100644 --- a/platform/linux-generic/odp_packet_flags.c +++ b/platform/linux-generic/odp_packet_flags.c @@ -8,137 +8,11 @@ #include <odp/api/packet_flags.h> #include <odp_packet_internal.h> -#define retflag(pkt, x) do { \ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); \ - return pkt_hdr->p.x; \ - } while (0) - #define setflag(pkt, x, v) do { \ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); \ pkt_hdr->p.x = (v) & 1; \ } while (0) -int odp_packet_has_error(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - return pkt_hdr->p.flags.all.error != 0; -} - -/* Get Input Flags */ - -int odp_packet_has_l2_error(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - /* L2 parsing is always done by default and hence - no additional check is required */ - return pkt_hdr->p.flags.snap_len_err; -} - -int odp_packet_has_l3(odp_packet_t pkt) -{ - retflag(pkt, input_flags.l3); -} - -int odp_packet_has_l3_error(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - return pkt_hdr->p.flags.ip_err; -} - -int odp_packet_has_l4(odp_packet_t pkt) -{ - retflag(pkt, input_flags.l4); -} - -int odp_packet_has_l4_error(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - return pkt_hdr->p.flags.tcp_err | pkt_hdr->p.flags.udp_err; -} - -int odp_packet_has_eth_bcast(odp_packet_t pkt) -{ - retflag(pkt, input_flags.eth_bcast); -} - -int odp_packet_has_eth_mcast(odp_packet_t pkt) -{ - retflag(pkt, input_flags.eth_mcast); -} - -int odp_packet_has_vlan(odp_packet_t pkt) -{ - retflag(pkt, input_flags.vlan); -} - -int odp_packet_has_vlan_qinq(odp_packet_t pkt) -{ - retflag(pkt, input_flags.vlan_qinq); -} - -int odp_packet_has_arp(odp_packet_t pkt) -{ - retflag(pkt, input_flags.arp); -} - -int odp_packet_has_ipv4(odp_packet_t pkt) -{ - retflag(pkt, input_flags.ipv4); -} - -int odp_packet_has_ipv6(odp_packet_t pkt) -{ - retflag(pkt, input_flags.ipv6); -} - -int odp_packet_has_ip_bcast(odp_packet_t pkt) -{ - retflag(pkt, input_flags.ip_bcast); -} - -int odp_packet_has_ip_mcast(odp_packet_t pkt) -{ - retflag(pkt, input_flags.ip_mcast); -} - -int odp_packet_has_ipfrag(odp_packet_t pkt) -{ - retflag(pkt, input_flags.ipfrag); -} - -int odp_packet_has_ipopt(odp_packet_t pkt) -{ - retflag(pkt, input_flags.ipopt); -} - -int odp_packet_has_udp(odp_packet_t pkt) -{ - retflag(pkt, input_flags.udp); -} - -int odp_packet_has_tcp(odp_packet_t pkt) -{ - retflag(pkt, input_flags.tcp); -} - -int odp_packet_has_sctp(odp_packet_t pkt) -{ - retflag(pkt, input_flags.sctp); -} - -int odp_packet_has_icmp(odp_packet_t pkt) -{ - retflag(pkt, input_flags.icmp); -} - -odp_packet_color_t odp_packet_color(odp_packet_t pkt) -{ - retflag(pkt, input_flags.color); -} - void odp_packet_color_set(odp_packet_t pkt, odp_packet_color_t color) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); @@ -146,23 +20,11 @@ void odp_packet_color_set(odp_packet_t pkt, odp_packet_color_t color) pkt_hdr->p.input_flags.color = color; } -odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - return !pkt_hdr->p.input_flags.nodrop; -} - void odp_packet_drop_eligible_set(odp_packet_t pkt, odp_bool_t drop) { setflag(pkt, input_flags.nodrop, !drop); } -int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt) -{ - retflag(pkt, flags.shaper_len_adj); -} - void odp_packet_shaper_len_adjust_set(odp_packet_t pkt, int8_t adj) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 899f10425..7c68b10c5 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -53,6 +53,10 @@ /* Max wait time supported to avoid potential overflow */ #define MAX_WAIT_TIME (UINT64_MAX / 1024) +/* One hour maximum aging timeout, no real limitations imposed by the implementation other than + * integer width, so just use some value. */ +#define MAX_TX_AGING_TMO_NS 3600000000000ULL + typedef struct { const void *user_ptr; odp_queue_t queue; @@ -624,6 +628,8 @@ int odp_pktio_config(odp_pktio_t hdl, const odp_pktio_config_t *config) return -1; } + entry->s.enabled.tx_aging = config->pktout.bit.aging_ena; + if (entry->s.ops->config) res = entry->s.ops->config(entry, config); @@ -650,6 +656,9 @@ int odp_pktio_start(odp_pktio_t hdl) ODP_ERR("Already started\n"); return -1; } + entry->s.parse_layer = pktio_cls_enabled(entry) ? + ODP_PROTO_LAYER_ALL : + entry->s.config.parser.layer; if (entry->s.ops->start) res = entry->s.ops->start(entry); if (!res) @@ -660,8 +669,8 @@ int odp_pktio_start(odp_pktio_t hdl) mode = entry->s.param.in_mode; if (mode == ODP_PKTIN_MODE_SCHED) { - unsigned int i; - unsigned int num = entry->s.num_in_queue; + uint32_t i; + uint32_t num = entry->s.num_in_queue; int index[num]; odp_queue_t odpq[num]; @@ -1182,95 +1191,6 @@ static int pktin_deq_multi(odp_queue_t queue, _odp_event_hdr_t *event_hdr[], return nbr; } -int _odp_sched_cb_pktin_poll_one(int pktio_index, - int rx_queue, - odp_event_t evt_tbl[]) -{ - int num_rx, num_pkts, i; - pktio_entry_t *entry = pktio_entry_by_index(pktio_index); - odp_packet_t pkt; - odp_packet_hdr_t *pkt_hdr; - odp_pool_t pool = ODP_POOL_INVALID; - odp_packet_t packets[QUEUE_MULTI_MAX]; - odp_queue_t queue; - odp_bool_t vector_enabled = entry->s.in_queue[rx_queue].vector.enable; - uint32_t num = QUEUE_MULTI_MAX; - cos_t *cos_hdr = NULL; - - if (odp_unlikely(entry->s.state != PKTIO_STATE_STARTED)) { - if (entry->s.state < PKTIO_STATE_ACTIVE || - entry->s.state == PKTIO_STATE_STOP_PENDING) - return -1; - - ODP_DBG("interface not started\n"); - return 0; - } - - if (vector_enabled) { - /* Make sure all packets will fit into a single packet vector */ - if (entry->s.in_queue[rx_queue].vector.max_size < num) - num = entry->s.in_queue[rx_queue].vector.max_size; - pool = entry->s.in_queue[rx_queue].vector.pool; - } - - ODP_ASSERT((unsigned int)rx_queue < entry->s.num_in_queue); - num_pkts = entry->s.ops->recv(entry, rx_queue, packets, num); - - num_rx = 0; - for (i = 0; i < num_pkts; i++) { - pkt = packets[i]; - pkt_hdr = packet_hdr(pkt); - if (odp_unlikely(pkt_hdr->p.input_flags.dst_queue)) { - odp_event_t event = odp_packet_to_event(pkt); - uint16_t cos_idx = pkt_hdr->cos; - - queue = pkt_hdr->dst_queue; - - if (cos_idx != CLS_COS_IDX_NONE) { - /* Packets from classifier */ - cos_hdr = _odp_cos_entry_from_idx(cos_idx); - - if (cos_hdr->s.vector.enable) { - packet_vector_enq_cos(queue, &event, 1, cos_hdr); - continue; - } - } else if (vector_enabled) { - /* Packets from inline IPsec */ - packet_vector_enq(queue, &event, 1, pool); - continue; - } - - if (odp_unlikely(odp_queue_enq(queue, event))) { - /* Queue full? */ - odp_packet_free(pkt); - if (cos_idx != CLS_COS_IDX_NONE) - _odp_cos_queue_stats_add(cos_hdr, queue, 0, 1); - else - odp_atomic_inc_u64(&entry->s.stats_extra.in_discards); - } else { - if (cos_idx != CLS_COS_IDX_NONE) - _odp_cos_queue_stats_add(cos_hdr, queue, 1, 0); - } - } else { - evt_tbl[num_rx++] = odp_packet_to_event(pkt); - } - } - - /* Create packet vector */ - if (vector_enabled && num_rx > 0) { - odp_packet_vector_t pktv = packet_vector_create((odp_packet_t *)evt_tbl, - num_rx, pool); - - if (odp_unlikely(pktv == ODP_PACKET_VECTOR_INVALID)) - return 0; - - evt_tbl[0] = odp_packet_vector_to_event(pktv); - return 1; - } - - return num_rx; -} - int _odp_sched_cb_pktin_poll(int pktio_index, int pktin_index, _odp_event_hdr_t *hdr_tbl[], int num) { @@ -1886,6 +1806,9 @@ int odp_pktio_capability(odp_pktio_t pktio, odp_pktio_capability_t *capa) capa->tx_compl.queue_type_sched = 1; capa->tx_compl.queue_type_plain = 1; capa->tx_compl.mode_all = 1; + + capa->config.pktout.bit.aging_ena = 1; + capa->max_tx_aging_tmo_ns = MAX_TX_AGING_TMO_NS; } /* Packet vector generation is common for all pktio types */ @@ -2246,8 +2169,7 @@ int odp_pktin_queue_config(odp_pktio_t pktio, pktio_entry_t *entry; odp_pktin_mode_t mode; odp_pktio_capability_t capa; - unsigned int num_queues; - unsigned int i; + uint32_t num_queues, i; int rc; odp_queue_t queue; odp_pktin_queue_param_t default_param; @@ -2410,7 +2332,7 @@ int _odp_pktio_pktout_tm_config(odp_pktio_t pktio_hdl, bool pktio_started = false; odp_pktout_mode_t mode; pktio_entry_t *entry; - unsigned int i; + uint32_t i; int rc = 0; odp_pktout_queue_param_init(¶m); @@ -2480,8 +2402,7 @@ int odp_pktout_queue_config(odp_pktio_t pktio, pktio_entry_t *entry; odp_pktout_mode_t mode; odp_pktio_capability_t capa; - unsigned int num_queues; - unsigned int i; + uint32_t num_queues, i; int rc; odp_pktout_queue_param_t default_param; @@ -2847,18 +2768,17 @@ int odp_pktin_recv_tmo(odp_pktin_queue_t queue, odp_packet_t packets[], int num, } } -int odp_pktin_recv_mq_tmo(const odp_pktin_queue_t queues[], unsigned int num_q, - unsigned int *from, odp_packet_t packets[], int num, - uint64_t wait) +int odp_pktin_recv_mq_tmo(const odp_pktin_queue_t queues[], uint32_t num_q, uint32_t *from, + odp_packet_t packets[], int num, uint64_t wait) { - unsigned int i; + uint32_t i; int ret; odp_time_t t1, t2; struct timespec ts; int started = 0; uint64_t sleep_round = 0; int trial_successful = 0; - unsigned int lfrom = 0; + uint32_t lfrom = 0; for (i = 0; i < num_q; i++) { ret = odp_pktin_recv(queues[i], packets, num); diff --git a/platform/linux-generic/odp_parse.c b/platform/linux-generic/odp_parse.c new file mode 100644 index 000000000..2342a7e49 --- /dev/null +++ b/platform/linux-generic/odp_parse.c @@ -0,0 +1,471 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2019-2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_parse_internal.h> +#include <odp_chksum_internal.h> +#include <protocols/eth.h> +#include <protocols/ip.h> +#include <protocols/sctp.h> +#include <protocols/tcp.h> +#include <protocols/udp.h> +#include <odp/api/hash.h> +#include <odp/api/packet_io.h> +#include <odp/api/packet_types.h> +#include <stdint.h> +#include <string.h> + +/** Parser helper function for Ethernet packets + * + * Requires up to PARSE_ETH_BYTES bytes of contiguous packet data. + */ +uint16_t _odp_parse_eth(packet_parser_t *prs, const uint8_t **parseptr, + uint32_t *offset, uint32_t frame_len) +{ + uint16_t ethtype; + const _odp_ethhdr_t *eth; + uint16_t macaddr0, macaddr2, macaddr4; + const _odp_vlanhdr_t *vlan; + _odp_packet_input_flags_t input_flags; + + input_flags.all = 0; + input_flags.l2 = 1; + input_flags.eth = 1; + + eth = (const _odp_ethhdr_t *)*parseptr; + + /* Detect jumbo frames */ + if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX)) + input_flags.jumbo = 1; + + /* Handle Ethernet broadcast/multicast addresses */ + macaddr0 = odp_be_to_cpu_16(*((const uint16_t *)(const void *)eth)); + if (odp_unlikely((macaddr0 & 0x0100) == 0x0100)) + input_flags.eth_mcast = 1; + + if (odp_unlikely(macaddr0 == 0xffff)) { + macaddr2 = + odp_be_to_cpu_16(*((const uint16_t *) + (const void *)eth + 1)); + macaddr4 = + odp_be_to_cpu_16(*((const uint16_t *) + (const void *)eth + 2)); + + if ((macaddr2 == 0xffff) && (macaddr4 == 0xffff)) + input_flags.eth_bcast = 1; + } + + /* Get Ethertype */ + ethtype = odp_be_to_cpu_16(eth->type); + *offset += sizeof(*eth); + *parseptr += sizeof(*eth); + + /* Check for SNAP vs. DIX */ + if (odp_unlikely(ethtype < _ODP_ETH_LEN_MAX)) { + input_flags.snap = 1; + if (ethtype > frame_len - *offset) { + prs->flags.snap_len_err = 1; + ethtype = 0; + goto error; + } + ethtype = odp_be_to_cpu_16(*((const uint16_t *)(uintptr_t) + (*parseptr + 6))); + *offset += 8; + *parseptr += 8; + } + + /* Parse the VLAN header(s), if present */ + if (odp_unlikely(ethtype == _ODP_ETHTYPE_VLAN_OUTER)) { + input_flags.vlan_qinq = 1; + input_flags.vlan = 1; + + vlan = (const _odp_vlanhdr_t *)*parseptr; + ethtype = odp_be_to_cpu_16(vlan->type); + *offset += sizeof(_odp_vlanhdr_t); + *parseptr += sizeof(_odp_vlanhdr_t); + } + + if (ethtype == _ODP_ETHTYPE_VLAN) { + input_flags.vlan = 1; + vlan = (const _odp_vlanhdr_t *)*parseptr; + ethtype = odp_be_to_cpu_16(vlan->type); + *offset += sizeof(_odp_vlanhdr_t); + *parseptr += sizeof(_odp_vlanhdr_t); + } + + /* + * The packet was too short for what we parsed. We just give up + * entirely without trying to parse what fits in the packet. + */ + if (odp_unlikely(*offset > frame_len)) { + input_flags.all = 0; + input_flags.l2 = 1; + ethtype = 0; + } + +error: + prs->input_flags.all |= input_flags.all; + + return ethtype; +} + +/** + * Parser helper function for IPv4 + * + * Requires up to PARSE_IPV4_BYTES bytes of contiguous packet data. + */ +static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, + uint32_t *offset, uint32_t frame_len, + odp_proto_chksums_t chksums, + uint64_t *l4_part_sum) +{ + const _odp_ipv4hdr_t *ipv4 = (const _odp_ipv4hdr_t *)*parseptr; + uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr); + uint32_t l3_len = odp_be_to_cpu_16(ipv4->tot_len); + uint16_t frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); + uint8_t ver = _ODP_IPV4HDR_VER(ipv4->ver_ihl); + uint8_t ihl = _ODP_IPV4HDR_IHL(ipv4->ver_ihl); + + if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN || + ver != 4 || + sizeof(*ipv4) > frame_len - *offset || + (l3_len > frame_len - *offset))) { + prs->flags.ip_err = 1; + return 0; + } + + if (chksums.chksum.ipv4) { + prs->input_flags.l3_chksum_done = 1; + if (chksum_finalize(chksum_partial(ipv4, ihl * 4, 0)) != 0xffff) { + prs->flags.ip_err = 1; + prs->flags.l3_chksum_err = 1; + return 0; + } + } + + *offset += ihl * 4; + *parseptr += ihl * 4; + + if (chksums.chksum.udp || chksums.chksum.tcp) + *l4_part_sum = chksum_partial((const uint8_t *)&ipv4->src_addr, + 2 * _ODP_IPV4ADDR_LEN, 0); + + if (odp_unlikely(ihl > _ODP_IPV4HDR_IHL_MIN)) + prs->input_flags.ipopt = 1; + + /* A packet is a fragment if: + * "more fragments" flag is set (all fragments except the last) + * OR + * "fragment offset" field is nonzero (all fragments except the first) + */ + if (odp_unlikely(_ODP_IPV4HDR_IS_FRAGMENT(frag_offset))) + prs->input_flags.ipfrag = 1; + + /* Handle IPv4 broadcast / multicast */ + if (odp_unlikely(dstaddr == 0xffffffff)) + prs->input_flags.ip_bcast = 1; + + if (odp_unlikely((dstaddr >> 28) == 0xe)) + prs->input_flags.ip_mcast = 1; + + return ipv4->proto; +} + +/** + * Parser helper function for IPv6 + * + * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data. + */ +static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, + uint32_t *offset, uint32_t frame_len, + uint32_t seg_len, + odp_proto_chksums_t chksums, + uint64_t *l4_part_sum) +{ + const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr; + const _odp_ipv6hdr_ext_t *ipv6ext; + uint32_t dstaddr0 = odp_be_to_cpu_32(ipv6->dst_addr.u8[0]); + uint32_t l3_len = odp_be_to_cpu_16(ipv6->payload_len) + + _ODP_IPV6HDR_LEN; + + /* Basic sanity checks on IPv6 header */ + if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || + sizeof(*ipv6) > frame_len - *offset || + l3_len > frame_len - *offset)) { + prs->flags.ip_err = 1; + return 0; + } + + /* IPv6 broadcast / multicast flags */ + prs->input_flags.ip_mcast = (dstaddr0 & 0xff000000) == 0xff000000; + prs->input_flags.ip_bcast = 0; + + /* Skip past IPv6 header */ + *offset += sizeof(_odp_ipv6hdr_t); + *parseptr += sizeof(_odp_ipv6hdr_t); + + if (chksums.chksum.udp || chksums.chksum.tcp) + *l4_part_sum = chksum_partial((const uint8_t *)&ipv6->src_addr, + 2 * _ODP_IPV6ADDR_LEN, 0); + + /* Skip past any IPv6 extension headers */ + if (ipv6->next_hdr == _ODP_IPPROTO_HOPOPTS || + ipv6->next_hdr == _ODP_IPPROTO_ROUTE) { + prs->input_flags.ipopt = 1; + + do { + ipv6ext = (const _odp_ipv6hdr_ext_t *)*parseptr; + uint16_t extlen = 8 + ipv6ext->ext_len * 8; + + *offset += extlen; + *parseptr += extlen; + } while ((ipv6ext->next_hdr == _ODP_IPPROTO_HOPOPTS || + ipv6ext->next_hdr == _ODP_IPPROTO_ROUTE) && + *offset < seg_len); + + if (*offset >= prs->l3_offset + + odp_be_to_cpu_16(ipv6->payload_len)) { + prs->flags.ip_err = 1; + return 0; + } + + if (ipv6ext->next_hdr == _ODP_IPPROTO_FRAG) + prs->input_flags.ipfrag = 1; + + return ipv6ext->next_hdr; + } + + if (odp_unlikely(ipv6->next_hdr == _ODP_IPPROTO_FRAG)) { + prs->input_flags.ipopt = 1; + prs->input_flags.ipfrag = 1; + } + + return ipv6->next_hdr; +} + +/** + * Parser helper function for TCP + * + * Requires PARSE_TCP_BYTES bytes of contiguous packet data. + */ +static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, + uint16_t tcp_len, + odp_proto_chksums_t chksums, + uint64_t *l4_part_sum) +{ + const _odp_tcphdr_t *tcp = (const _odp_tcphdr_t *)*parseptr; + uint32_t len = tcp->hl * 4; + + if (odp_unlikely(tcp->hl < sizeof(_odp_tcphdr_t) / sizeof(uint32_t))) + prs->flags.tcp_err = 1; + + if (chksums.chksum.tcp && + !prs->input_flags.ipfrag) { + *l4_part_sum += odp_cpu_to_be_16(tcp_len); +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN + *l4_part_sum += _ODP_IPPROTO_TCP; +#else + *l4_part_sum += _ODP_IPPROTO_TCP << 8; +#endif + } + + *parseptr += len; +} + +/** + * Parser helper function for UDP + * + * Requires PARSE_UDP_BYTES bytes of contiguous packet data. + */ +static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, + odp_proto_chksums_t chksums, + uint64_t *l4_part_sum) +{ + const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr; + uint32_t udplen = odp_be_to_cpu_16(udp->length); + uint16_t ipsec_port = odp_cpu_to_be_16(_ODP_UDP_IPSEC_PORT); + + if (odp_unlikely(udplen < sizeof(_odp_udphdr_t))) { + prs->flags.udp_err = 1; + return; + } + + if (chksums.chksum.udp && + !prs->input_flags.ipfrag) { + if (udp->chksum == 0) { + prs->input_flags.l4_chksum_done = 1; + prs->flags.l4_chksum_err = + (prs->input_flags.ipv4 != 1); + } else { + *l4_part_sum += udp->length; +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN + *l4_part_sum += _ODP_IPPROTO_UDP; +#else + *l4_part_sum += _ODP_IPPROTO_UDP << 8; +#endif + } + prs->input_flags.udp_chksum_zero = (udp->chksum == 0); + } + + if (odp_unlikely(ipsec_port == udp->dst_port && udplen > 4)) { + uint32_t val; + + memcpy(&val, udp + 1, 4); + if (val != 0) { + prs->input_flags.ipsec = 1; + prs->input_flags.ipsec_udp = 1; + } + } + + *parseptr += sizeof(_odp_udphdr_t); +} + +/** + * Parser helper function for SCTP + * + * Requires PARSE_SCTP_BYTES bytes of contiguous packet data. + */ +static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, + uint16_t sctp_len, + odp_proto_chksums_t chksums, + uint64_t *l4_part_sum) +{ + if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) { + prs->flags.sctp_err = 1; + return; + } + + if (chksums.chksum.sctp && + !prs->input_flags.ipfrag) { + const _odp_sctphdr_t *sctp = + (const _odp_sctphdr_t *)*parseptr; + uint32_t crc = ~0; + uint32_t zero = 0; + + crc = odp_hash_crc32c(sctp, sizeof(*sctp) - 4, crc); + crc = odp_hash_crc32c(&zero, 4, crc); + *l4_part_sum = crc; + } + + *parseptr += sizeof(_odp_sctphdr_t); +} + +/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */ +int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, + const uint8_t *parseptr, uint32_t offset, + uint32_t frame_len, uint32_t seg_len, + int layer, uint16_t ethtype, + odp_proto_chksums_t chksums, + uint64_t *l4_part_sum, + odp_pktin_config_opt_t opt) +{ + uint8_t ip_proto; + + prs->l3_offset = offset; + + if (odp_unlikely(layer <= ODP_PROTO_LAYER_L2)) + return prs->flags.all.error != 0; + + /* Set l3 flag only for known ethtypes */ + prs->input_flags.l3 = 1; + + /* Parse Layer 3 headers */ + switch (ethtype) { + case _ODP_ETHTYPE_IPV4: + prs->input_flags.ipv4 = 1; + ip_proto = parse_ipv4(prs, &parseptr, &offset, frame_len, + chksums, l4_part_sum); + prs->l4_offset = offset; + if (prs->flags.ip_err && opt.bit.drop_ipv4_err) + return -1; /* drop */ + break; + + case _ODP_ETHTYPE_IPV6: + prs->input_flags.ipv6 = 1; + ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len, + seg_len, chksums, l4_part_sum); + prs->l4_offset = offset; + if (prs->flags.ip_err && opt.bit.drop_ipv6_err) + return -1; /* drop */ + break; + + case _ODP_ETHTYPE_ARP: + prs->input_flags.arp = 1; + ip_proto = 255; /* Reserved invalid by IANA */ + break; + + default: + prs->input_flags.l3 = 0; + ip_proto = 255; /* Reserved invalid by IANA */ + } + + if (layer == ODP_PROTO_LAYER_L3) + return prs->flags.all.error != 0; + + /* Set l4 flag only for known ip_proto */ + prs->input_flags.l4 = 1; + + /* Parse Layer 4 headers */ + switch (ip_proto) { + case _ODP_IPPROTO_ICMPV4: + /* Fall through */ + + case _ODP_IPPROTO_ICMPV6: + prs->input_flags.icmp = 1; + break; + + case _ODP_IPPROTO_IPIP: + /* Do nothing */ + break; + + case _ODP_IPPROTO_TCP: + if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len)) + return -1; + prs->input_flags.tcp = 1; + parse_tcp(prs, &parseptr, frame_len - prs->l4_offset, chksums, + l4_part_sum); + if (prs->flags.tcp_err && opt.bit.drop_tcp_err) + return -1; /* drop */ + break; + + case _ODP_IPPROTO_UDP: + if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len)) + return -1; + prs->input_flags.udp = 1; + parse_udp(prs, &parseptr, chksums, l4_part_sum); + if (prs->flags.udp_err && opt.bit.drop_udp_err) + return -1; /* drop */ + break; + + case _ODP_IPPROTO_AH: + prs->input_flags.ipsec = 1; + prs->input_flags.ipsec_ah = 1; + break; + + case _ODP_IPPROTO_ESP: + prs->input_flags.ipsec = 1; + prs->input_flags.ipsec_esp = 1; + break; + + case _ODP_IPPROTO_SCTP: + prs->input_flags.sctp = 1; + parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, chksums, + l4_part_sum); + if (prs->flags.sctp_err && opt.bit.drop_sctp_err) + return -1; /* drop */ + break; + + case _ODP_IPPROTO_NO_NEXT: + prs->input_flags.no_next_hdr = 1; + break; + + default: + prs->input_flags.l4 = 0; + break; + } + + return prs->flags.all.error != 0; +} diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index cca2d55f0..defdeb4fb 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -494,12 +494,7 @@ static void init_buffers(pool_t *pool) for (i = 0; i < pool->num + skipped_blocks ; i++) { int skip = 0; - - addr = &pool->base_addr[(i * pool->block_size) + - pool->block_offset]; - event_hdr = addr; - buf_hdr = addr; - pkt_hdr = addr; + addr = &pool->base_addr[i * pool->block_size]; /* Skip packet buffers which cross huge page boundaries. Some * NICs cannot handle buffers which cross page boundaries. */ @@ -517,6 +512,12 @@ static void init_buffers(pool_t *pool) skip = 1; } } + + addr = (uint8_t *)addr + pool->block_offset; + event_hdr = addr; + buf_hdr = addr; + pkt_hdr = addr; + if (pool->uarea_size) uarea = &pool->uarea_base_addr[(i - skipped_blocks) * pool->uarea_size]; diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c index f5c3830e8..58959df31 100644 --- a/platform/linux-generic/odp_queue_lf.c +++ b/platform/linux-generic/odp_queue_lf.c @@ -13,6 +13,7 @@ #include <odp_debug_internal.h> #include <odp_event_internal.h> #include <odp_queue_basic_internal.h> +#include <odp_types_internal.h> #include <string.h> #include <stdio.h> @@ -24,9 +25,7 @@ #ifdef __SIZEOF_INT128__ -__extension__ typedef unsigned __int128 u128_t; - -static inline void lockfree_zero_u128(u128_t *atomic) +static inline void lockfree_zero_u128(_odp_u128_t *atomic) { __atomic_store_n(atomic, 0, __ATOMIC_RELAXED); } @@ -40,21 +39,21 @@ static inline void lockfree_zero_u128(u128_t *atomic) * So, these are never actually used. */ typedef struct ODP_ALIGNED(16) { uint64_t u64[2]; -} u128_t; +} _odp_u128_t; -static inline u128_t lockfree_load_u128(u128_t *atomic) +static inline _odp_u128_t lockfree_load_u128(_odp_u128_t *atomic) { return *atomic; } -static inline void lockfree_zero_u128(u128_t *atomic) +static inline void lockfree_zero_u128(_odp_u128_t *atomic) { atomic->u64[0] = 0; atomic->u64[1] = 0; } -static inline int lockfree_cas_acq_rel_u128(u128_t *atomic, u128_t old_val, - u128_t new_val) +static inline int lockfree_cas_acq_rel_u128(_odp_u128_t *atomic, _odp_u128_t old_val, + _odp_u128_t new_val) { if (atomic->u64[0] == old_val.u64[0] && atomic->u64[1] == old_val.u64[1]) { @@ -75,7 +74,7 @@ static inline int lockfree_check_u128(void) /* Node in lock-free ring */ typedef union { - u128_t u128; + _odp_u128_t u128; struct { /* Data with lowest counter value is the head. Empty node has diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 5343c2834..7482d776d 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -817,23 +817,19 @@ events_dequeued: } /* Ingress queue empty => poll pktio RX queue */ - odp_event_t rx_evts[QUEUE_MULTI_MAX]; - int num_rx = _odp_sched_cb_pktin_poll_one(elem->pktio_idx, - elem->rx_queue, - rx_evts); + _odp_event_hdr_t *rx_evts[QUEUE_MULTI_MAX]; + int num_rx = _odp_sched_cb_pktin_poll(elem->pktio_idx, elem->rx_queue, + rx_evts, QUEUE_MULTI_MAX); + if (odp_likely(num_rx > 0)) { num = num_rx < num_evts ? num_rx : num_evts; for (i = 0; i < num; i++) { /* Return events directly to caller */ - ev[i] = rx_evts[i]; + ev[i] = _odp_event_from_hdr(rx_evts[i]); } if (num_rx > num) { /* Events remain, enqueue them */ - _odp_event_hdr_t *events[QUEUE_MULTI_MAX]; - - for (i = num; i < num_rx; i++) - events[i] = _odp_event_hdr(rx_evts[i]); - i = _odp_queue_enq_sp(elem, &events[num], num_rx - num); + i = _odp_queue_enq_sp(elem, &rx_evts[num], num_rx - num); /* Enqueue must succeed as the queue was empty */ ODP_ASSERT(i == num_rx - num); } diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index f5aba1a88..558c3f4fe 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -13,6 +13,7 @@ #pragma GCC diagnostic ignored "-Wzero-length-bounds" #endif +#include <odp/api/packet.h> #include <odp/api/ticketlock.h> #include <odp/api/thread.h> #include <odp/api/plat/thread_inlines.h> diff --git a/platform/linux-generic/odp_time.c b/platform/linux-generic/odp_time.c index d908f02cb..b591d712a 100644 --- a/platform/linux-generic/odp_time.c +++ b/platform/linux-generic/odp_time.c @@ -145,15 +145,6 @@ static inline void time_wait_until(odp_time_t time) } while (odp_time_cmp(time, cur) > 0); } -uint64_t odp_time_diff_ns(odp_time_t t2, odp_time_t t1) -{ - odp_time_t time; - - time.u64 = t2.u64 - t1.u64; - - return odp_time_to_ns(time); -} - odp_time_t odp_time_local_from_ns(uint64_t ns) { return time_from_ns(ns); diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index d1d4a5bc8..1c54ab740 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -13,49 +13,52 @@ */ #include <odp_posix_extensions.h> -#include <errno.h> -#include <stdlib.h> -#include <time.h> -#include <signal.h> -#include <pthread.h> -#include <unistd.h> -#include <sys/syscall.h> -#include <inttypes.h> -#include <string.h> - -#include <odp/api/std.h> #include <odp/api/align.h> -#include <odp_align_internal.h> #include <odp/api/atomic.h> -#include <odp/api/plat/atomic_inlines.h> -#include <odp_atomic_internal.h> -#include <odp/api/buffer.h> #include <odp/api/cpu.h> -#include <odp/api/pool.h> -#include <odp_pool_internal.h> #include <odp/api/debug.h> -#include <odp_debug_internal.h> #include <odp/api/event.h> #include <odp/api/hints.h> -#include <odp_init_internal.h> -#include <odp_errno_define.h> +#include <odp/api/pool.h> #include <odp/api/queue.h> #include <odp/api/shared_memory.h> #include <odp/api/spinlock.h> -#include <odp/api/std_types.h> +#include <odp/api/std.h> #include <odp/api/sync.h> #include <odp/api/time.h> -#include <odp/api/plat/time_inlines.h> #include <odp/api/timer.h> + +/* Inlined API functions */ +#include <odp/api/plat/atomic_inlines.h> +#include <odp/api/plat/event_inlines.h> +#include <odp/api/plat/queue_inlines.h> +#include <odp/api/plat/time_inlines.h> +#include <odp/api/plat/timer_inlines.h> + +#include <odp/api/plat/timer_inline_types.h> + +#include <odp_align_internal.h> +#include <odp_atomic_internal.h> +#include <odp_debug_internal.h> +#include <odp_errno_define.h> +#include <odp_event_internal.h> +#include <odp_global_data.h> +#include <odp_init_internal.h> #include <odp_libconfig_internal.h> +#include <odp_pool_internal.h> #include <odp_queue_if.h> #include <odp_timer_internal.h> -#include <odp/api/plat/queue_inlines.h> -#include <odp_global_data.h> -#include <odp_event_internal.h> +#include <odp_types_internal.h> -/* Inlined API functions */ -#include <odp/api/plat/event_inlines.h> +#include <errno.h> +#include <inttypes.h> +#include <pthread.h> +#include <signal.h> +#include <stdlib.h> +#include <string.h> +#include <sys/syscall.h> +#include <time.h> +#include <unistd.h> /* One divided by one nanosecond in Hz */ #define GIGA_HZ 1000000000 @@ -94,6 +97,18 @@ #define IDX2LOCK(idx) (&timer_global->locks[(idx) % NUM_LOCKS]) #endif +#include <odp/visibility_begin.h> + +/* Fill in timeout header field offsets for inline functions */ +const _odp_timeout_inline_offset_t +_odp_timeout_inline_offset ODP_ALIGNED_CACHE = { + .expiration = offsetof(odp_timeout_hdr_t, expiration), + .timer = offsetof(odp_timeout_hdr_t, timer), + .user_ptr = offsetof(odp_timeout_hdr_t, user_ptr) +}; + +#include <odp/visibility_end.h> + typedef struct #ifdef ODP_ATOMIC_U128 ODP_ALIGNED(16) /* 16-byte atomic operations need properly aligned addresses */ @@ -689,7 +704,7 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, /* Atomic CAS will fail if we experienced torn reads, * retry update sequence until CAS succeeds */ } while (!_odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb, - (_uint128_t *)&old, (_uint128_t *)&new, + (_odp_u128_t *)&old, (_odp_u128_t *)&new, _ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX)); #elif __GCC_ATOMIC_LLONG_LOCK_FREE >= 2 && \ defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 @@ -760,8 +775,8 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, /* We are releasing the new timeout event to some other * thread */ _odp_atomic_u128_xchg_mm((_odp_atomic_u128_t *)tb, - (_uint128_t *)&new, - (_uint128_t *)&old, + (_odp_u128_t *)&new, + (_odp_u128_t *)&old, _ODP_MEMMODEL_ACQ_RLS); old_event = old.tmo_event; #else @@ -804,7 +819,7 @@ static odp_event_t timer_set_unused(timer_pool_t *tp, uint32_t idx) new.tmo_event = ODP_EVENT_INVALID; _odp_atomic_u128_xchg_mm((_odp_atomic_u128_t *)tb, - (_uint128_t *)&new, (_uint128_t *)&old, + (_odp_u128_t *)&new, (_odp_u128_t *)&old, _ODP_MEMMODEL_RLX); old_event = old.tmo_event; #else @@ -858,8 +873,8 @@ static odp_event_t timer_cancel(timer_pool_t *tp, uint32_t idx) /* Atomic CAS will fail if we experienced torn reads, * retry update sequence until CAS succeeds */ } while (!_odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb, - (_uint128_t *)&old, - (_uint128_t *)&new, + (_odp_u128_t *)&old, + (_odp_u128_t *)&new, _ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX)); old_event = old.tmo_event; @@ -916,7 +931,7 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) new.tmo_event = ODP_EVENT_INVALID; int succ = _odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb, - (_uint128_t *)&old, (_uint128_t *)&new, + (_odp_u128_t *)&old, (_odp_u128_t *)&new, _ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX); if (succ) tmo_event = old.tmo_event; @@ -1468,22 +1483,6 @@ void odp_timer_pool_destroy(odp_timer_pool_t tpid) odp_timer_pool_del(timer_pool_from_hdl(tpid)); } -uint64_t odp_timer_tick_to_ns(odp_timer_pool_t tpid, uint64_t ticks) -{ - (void)tpid; - - /* Timer ticks in API are nsec */ - return ticks; -} - -uint64_t odp_timer_ns_to_tick(odp_timer_pool_t tpid, uint64_t ns) -{ - (void)tpid; - - /* Timer ticks in API are nsec */ - return ns; -} - uint64_t odp_timer_current_tick(odp_timer_pool_t tpid) { timer_pool_t *tp = timer_pool_from_hdl(tpid); @@ -1812,19 +1811,6 @@ uint64_t odp_timer_to_u64(odp_timer_t hdl) return _odp_pri(hdl); } -odp_timeout_t odp_timeout_from_event(odp_event_t ev) -{ - /* This check not mandated by the API specification */ - if (odp_event_type(ev) != ODP_EVENT_TIMEOUT) - ODP_ABORT("Event not a timeout"); - return (odp_timeout_t)ev; -} - -odp_event_t odp_timeout_to_event(odp_timeout_t tmo) -{ - return (odp_event_t)tmo; -} - uint64_t odp_timeout_to_u64(odp_timeout_t tmo) { return _odp_pri(tmo); @@ -1847,21 +1833,6 @@ int odp_timeout_fresh(odp_timeout_t tmo) return hdr->expiration == (exp_tck & ~TMO_INACTIVE); } -odp_timer_t odp_timeout_timer(odp_timeout_t tmo) -{ - return timeout_hdr(tmo)->timer; -} - -uint64_t odp_timeout_tick(odp_timeout_t tmo) -{ - return timeout_hdr(tmo)->expiration; -} - -void *odp_timeout_user_ptr(odp_timeout_t tmo) -{ - return (void *)(uintptr_t)timeout_hdr(tmo)->user_ptr; -} - odp_timeout_t odp_timeout_alloc(odp_pool_t pool_hdl) { odp_event_t event; diff --git a/platform/linux-generic/odp_timer_api.c b/platform/linux-generic/odp_timer_api.c new file mode 100644 index 000000000..cd657956b --- /dev/null +++ b/platform/linux-generic/odp_timer_api.c @@ -0,0 +1,11 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/timer.h> + +/* Non-inlined functions for ABI compat mode */ +#define _ODP_NO_INLINE +#include <odp/api/plat/timer_inlines.h> diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index fbf33e117..f02297ec9 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -55,45 +55,60 @@ _odp_int_name_kind_t PROFILE_TO_HANDLE_KIND[ODP_TM_NUM_PROFILES] = { static const pkt_desc_t EMPTY_PKT_DESC = { .word = 0 }; #define MAX_PRIORITIES ODP_TM_MAX_PRIORITIES -#define NUM_SHAPER_COLORS ODP_NUM_SHAPER_COLORS /* Shaper BW limits in bits/sec */ #define TM_MIN_SHAPER_BW 8000ULL #define TM_MAX_SHAPER_BW (100ULL * 1000ULL * 1000ULL * 1000ULL) +/* Possible values for running the shaper algorithm. TM_SHAPER_GREEN means that + * the traffic is within the commit specification (rate and burst size), + * TM_SHAPER_YELLOW means that the traffic is within the peak specification + * (rate and burst size) and TM_SHAPER_RED means that the traffic is exceeding + * both its commit and peak specifications. Note that packets can also have an + * assigned packet color of ODP_PACKET_GREEN, ODP_PACKET_YELLOW or + * ODP_PACKET_RED, which has a different meaning and purpose than the shaper + * colors. + */ +typedef enum { + TM_SHAPER_GREEN, TM_SHAPER_YELLOW, TM_SHAPER_RED +} tm_shaper_color_t; + +/* Number of enumeration values defined in tm_shaper_color_t type. */ +#define NUM_SHAPER_COLORS 3 + static const tm_prop_t basic_prop_tbl[MAX_PRIORITIES][NUM_SHAPER_COLORS] = { [0] = { - [ODP_TM_SHAPER_GREEN] = { 0, DECR_BOTH }, - [ODP_TM_SHAPER_YELLOW] = { 0, DECR_BOTH }, - [ODP_TM_SHAPER_RED] = { 0, DELAY_PKT } }, + [TM_SHAPER_GREEN] = { 0, DECR_BOTH }, + [TM_SHAPER_YELLOW] = { 0, DECR_BOTH }, + [TM_SHAPER_RED] = { 0, DELAY_PKT } }, [1] = { - [ODP_TM_SHAPER_GREEN] = { 1, DECR_BOTH }, - [ODP_TM_SHAPER_YELLOW] = { 1, DECR_BOTH }, - [ODP_TM_SHAPER_RED] = { 1, DELAY_PKT } }, + [TM_SHAPER_GREEN] = { 1, DECR_BOTH }, + [TM_SHAPER_YELLOW] = { 1, DECR_BOTH }, + [TM_SHAPER_RED] = { 1, DELAY_PKT } }, [2] = { - [ODP_TM_SHAPER_GREEN] = { 2, DECR_BOTH }, - [ODP_TM_SHAPER_YELLOW] = { 2, DECR_BOTH }, - [ODP_TM_SHAPER_RED] = { 2, DELAY_PKT } }, + [TM_SHAPER_GREEN] = { 2, DECR_BOTH }, + [TM_SHAPER_YELLOW] = { 2, DECR_BOTH }, + [TM_SHAPER_RED] = { 2, DELAY_PKT } }, [3] = { - [ODP_TM_SHAPER_GREEN] = { 3, DECR_BOTH }, - [ODP_TM_SHAPER_YELLOW] = { 3, DECR_BOTH }, - [ODP_TM_SHAPER_RED] = { 3, DELAY_PKT } }, + [TM_SHAPER_GREEN] = { 3, DECR_BOTH }, + [TM_SHAPER_YELLOW] = { 3, DECR_BOTH }, + [TM_SHAPER_RED] = { 3, DELAY_PKT } }, [4] = { - [ODP_TM_SHAPER_GREEN] = { 4, DECR_BOTH }, - [ODP_TM_SHAPER_YELLOW] = { 4, DECR_BOTH }, - [ODP_TM_SHAPER_RED] = { 4, DELAY_PKT } }, + [TM_SHAPER_GREEN] = { 4, DECR_BOTH }, + [TM_SHAPER_YELLOW] = { 4, DECR_BOTH }, + [TM_SHAPER_RED] = { 4, DELAY_PKT } }, [5] = { - [ODP_TM_SHAPER_GREEN] = { 5, DECR_BOTH }, - [ODP_TM_SHAPER_YELLOW] = { 5, DECR_BOTH }, - [ODP_TM_SHAPER_RED] = { 5, DELAY_PKT } }, + [TM_SHAPER_GREEN] = { 5, DECR_BOTH }, + [TM_SHAPER_YELLOW] = { 5, DECR_BOTH }, + [TM_SHAPER_RED] = { 5, DELAY_PKT } }, [6] = { - [ODP_TM_SHAPER_GREEN] = { 6, DECR_BOTH }, - [ODP_TM_SHAPER_YELLOW] = { 6, DECR_BOTH }, - [ODP_TM_SHAPER_RED] = { 6, DELAY_PKT } }, + [TM_SHAPER_GREEN] = { 6, DECR_BOTH }, + [TM_SHAPER_YELLOW] = { 6, DECR_BOTH }, + [TM_SHAPER_RED] = { 6, DELAY_PKT } }, [7] = { - [ODP_TM_SHAPER_GREEN] = { 7, DECR_BOTH }, - [ODP_TM_SHAPER_YELLOW] = { 7, DECR_BOTH }, - [ODP_TM_SHAPER_RED] = { 7, DELAY_PKT } } + [TM_SHAPER_GREEN] = { 7, DECR_BOTH }, + [TM_SHAPER_YELLOW] = { 7, DECR_BOTH }, + [TM_SHAPER_RED] = { 7, DELAY_PKT } } }; #define MAX_SHAPER_PROFILES 128 @@ -1076,30 +1091,30 @@ static odp_bool_t run_shaper(tm_system_t *tm_system, pkt_desc_t *pkt_desc, uint8_t priority) { - odp_tm_shaper_color_t shaper_color; + tm_shaper_color_t shaper_color; tm_shaper_params_t *shaper_params; odp_bool_t output_change; tm_prop_t propagation; shaper_params = shaper_obj->shaper_params; - shaper_color = ODP_TM_SHAPER_GREEN; + shaper_color = TM_SHAPER_GREEN; if (shaper_params) { update_shaper_elapsed_time(tm_system, shaper_params, shaper_obj); if (shaper_params->enabled) { if (0 < shaper_obj->commit_cnt) - shaper_color = ODP_TM_SHAPER_GREEN; + shaper_color = TM_SHAPER_GREEN; else if (!shaper_params->dual_rate) - shaper_color = ODP_TM_SHAPER_RED; + shaper_color = TM_SHAPER_RED; else if (shaper_obj->peak_cnt <= 0) - shaper_color = ODP_TM_SHAPER_RED; + shaper_color = TM_SHAPER_RED; else - shaper_color = ODP_TM_SHAPER_YELLOW; + shaper_color = TM_SHAPER_YELLOW; - if (shaper_color == ODP_TM_SHAPER_GREEN) + if (shaper_color == TM_SHAPER_GREEN) tm_system->shaper_green_cnt++; - else if (shaper_color == ODP_TM_SHAPER_YELLOW) + else if (shaper_color == TM_SHAPER_YELLOW) tm_system->shaper_yellow_cnt++; else tm_system->shaper_red_cnt++; @@ -1994,6 +2009,12 @@ static void tm_queue_cnts_decrement(tm_system_t *tm_system, odp_atomic_sub_u64(&queue_cnts->byte_cnt, frame_len); } +static inline void activate_packet_aging(odp_packet_hdr_t *pkt_hdr) +{ + if (odp_unlikely(pkt_hdr->p.flags.tx_aging)) + pkt_hdr->tx_aging_ns = pkt_hdr->tx_aging_ns + odp_time_global_ns(); +} + static int tm_enqueue(tm_system_t *tm_system, tm_queue_obj_t *tm_queue_obj, odp_packet_t pkt) @@ -2028,6 +2049,7 @@ static int tm_enqueue(tm_system_t *tm_system, if (tm_queue_obj->ordered_enqueue) _odp_sched_fn->order_lock(); + activate_packet_aging(packet_hdr(pkt)); rc = input_work_queue_append(tm_system, &work_item); if (tm_queue_obj->ordered_enqueue) @@ -2224,6 +2246,11 @@ static void tm_egress_marking(tm_system_t *tm_system, odp_packet_t odp_pkt) } } +static inline odp_bool_t is_packet_aged(odp_packet_hdr_t *pkt_hdr) +{ + return pkt_hdr->p.flags.tx_aging && pkt_hdr->tx_aging_ns < odp_time_global_ns(); +} + static void tm_send_pkt(tm_system_t *tm_system, uint32_t max_sends) { tm_queue_obj_t *tm_queue_obj; @@ -2250,9 +2277,13 @@ static void tm_send_pkt(tm_system_t *tm_system, uint32_t max_sends) tm_system->egress_pkt_desc = EMPTY_PKT_DESC; if (tm_system->egress.egress_kind == ODP_TM_EGRESS_PKT_IO) { - ret = odp_pktout_send(tm_system->pktout, &odp_pkt, 1); + pktio_entry = get_pktio_entry(tm_system->pktout.pktio); + if (odp_unlikely(_odp_pktio_tx_aging_enabled(pktio_entry) && + is_packet_aged(packet_hdr(odp_pkt)))) + ret = 0; /* Aged packet handled as a discard */ + else + ret = odp_pktout_send(tm_system->pktout, &odp_pkt, 1); if (odp_unlikely(ret != 1)) { - pktio_entry = get_pktio_entry(tm_system->pktout.pktio); if (odp_unlikely(_odp_pktio_tx_compl_enabled(pktio_entry))) _odp_pktio_allocate_and_send_tx_compl_events(pktio_entry, &odp_pkt, 1); @@ -2581,7 +2612,11 @@ static int tm_capabilities(odp_tm_capabilities_t capabilities[], cap_ptr->vlan_marking_supported = true; cap_ptr->ecn_marking_supported = true; cap_ptr->drop_prec_marking_supported = true; - cap_ptr->tm_queue_threshold = true; + + cap_ptr->tm_queue_threshold.byte = true; + cap_ptr->tm_queue_threshold.packet = true; + cap_ptr->tm_queue_threshold.byte_and_packet = true; + cap_ptr->tm_queue_query_flags = (ODP_TM_QUERY_PKT_CNT | ODP_TM_QUERY_BYTE_CNT | ODP_TM_QUERY_THRESHOLDS); @@ -2617,7 +2652,10 @@ static int tm_capabilities(odp_tm_capabilities_t capabilities[], per_level_cap->tm_node_dual_slope_supported = true; per_level_cap->fair_queuing_supported = true; per_level_cap->weights_supported = true; - per_level_cap->tm_node_threshold = true; + + per_level_cap->tm_node_threshold.byte = true; + per_level_cap->tm_node_threshold.packet = true; + per_level_cap->tm_node_threshold.byte_and_packet = true; } cap_ptr->queue_stats.counter.discards = 1; @@ -2691,7 +2729,11 @@ static void tm_system_capabilities_set(odp_tm_capabilities_t *cap_ptr, cap_ptr->ecn_marking_supported = req_ptr->ecn_marking_needed; cap_ptr->drop_prec_marking_supported = req_ptr->drop_prec_marking_needed; - cap_ptr->tm_queue_threshold = threshold; + + cap_ptr->tm_queue_threshold.byte = threshold; + cap_ptr->tm_queue_threshold.packet = threshold; + cap_ptr->tm_queue_threshold.byte_and_packet = threshold; + cap_ptr->tm_queue_query_flags = (ODP_TM_QUERY_PKT_CNT | ODP_TM_QUERY_BYTE_CNT | ODP_TM_QUERY_THRESHOLDS); @@ -2743,7 +2785,10 @@ static void tm_system_capabilities_set(odp_tm_capabilities_t *cap_ptr, per_level_cap->tm_node_dual_slope_supported = dual_slope; per_level_cap->fair_queuing_supported = true; per_level_cap->weights_supported = true; - per_level_cap->tm_node_threshold = threshold; + + per_level_cap->tm_node_threshold.byte = threshold; + per_level_cap->tm_node_threshold.packet = threshold; + per_level_cap->tm_node_threshold.byte_and_packet = threshold; } cap_ptr->queue_stats.counter.discards = 1; diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 815ec61b9..030560b0d 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -57,12 +57,14 @@ #include <rte_string_fns.h> #include <rte_version.h> -#if RTE_VERSION < RTE_VERSION_NUM(19, 8, 0, 0) - #define rte_ether_addr ether_addr - #define rte_ipv4_hdr ipv4_hdr - #define rte_ipv6_hdr ipv6_hdr - #define rte_tcp_hdr tcp_hdr - #define rte_udp_hdr udp_hdr +#if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0) + #define RTE_MBUF_F_RX_RSS_HASH PKT_RX_RSS_HASH + #define RTE_MBUF_F_TX_IPV4 PKT_TX_IPV4 + #define RTE_MBUF_F_TX_IPV6 PKT_TX_IPV6 + #define RTE_MBUF_F_TX_IP_CKSUM PKT_TX_IP_CKSUM + #define RTE_MBUF_F_TX_UDP_CKSUM PKT_TX_UDP_CKSUM + #define RTE_MBUF_F_TX_TCP_CKSUM PKT_TX_TCP_CKSUM + #define RTE_MEMPOOL_REGISTER_OPS MEMPOOL_REGISTER_OPS #endif /* NUMA is not supported on all platforms */ @@ -72,17 +74,7 @@ #define numa_num_configured_nodes() 1 #endif -#if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 0) -#define rte_log_set_global_level rte_set_log_level -#endif - -/* Release notes v19.11: "Changed the mempool allocation behaviour - * so that objects no longer cross pages by default" */ -#if RTE_VERSION >= RTE_VERSION_NUM(19, 11, 0, 0) -#define MEMPOOL_FLAGS MEMPOOL_F_NO_IOVA_CONTIG -#else #define MEMPOOL_FLAGS 0 -#endif #if _ODP_DPDK_ZERO_COPY ODP_STATIC_ASSERT(CONFIG_PACKET_HEADROOM == RTE_PKTMBUF_HEADROOM, @@ -107,11 +99,7 @@ ODP_STATIC_ASSERT((DPDK_NB_MBUF % DPDK_MEMPOOL_CACHE_SIZE == 0) && #define DPDK_MIN_RX_BURST 4 /* Limits for setting link MTU */ -#if RTE_VERSION >= RTE_VERSION_NUM(19, 11, 0, 0) #define DPDK_MTU_MIN (RTE_ETHER_MIN_MTU + _ODP_ETHHDR_LEN) -#else -#define DPDK_MTU_MIN (68 + _ODP_ETHHDR_LEN) -#endif #define DPDK_MTU_MAX (9000 + _ODP_ETHHDR_LEN) /** DPDK runtime configuration options */ @@ -334,12 +322,16 @@ static void pktmbuf_init(struct rte_mempool *mp, void *opaque_arg ODP_UNUSED, static struct rte_mempool *mbuf_pool_create(const char *name, pool_t *pool_entry) { + odp_shm_info_t shm_info; struct rte_mempool *mp = NULL; struct rte_pktmbuf_pool_private mbp_priv; struct rte_mempool_objsz sz; unsigned int elt_size = pool_entry->dpdk_elt_size; - unsigned int num = pool_entry->num; + unsigned int num = pool_entry->num, populated = 0; uint32_t total_size; + uint64_t page_size, offset = 0, remainder = 0; + uint8_t *addr; + int ret; if (!(pool_entry->mem_from_huge_pages)) { ODP_ERR("DPDK requires memory is allocated from huge pages\n"); @@ -348,11 +340,17 @@ static struct rte_mempool *mbuf_pool_create(const char *name, if (pool_entry->seg_len < RTE_MBUF_DEFAULT_BUF_SIZE) { ODP_ERR("Some NICs need at least %dB buffers to not segment " - "standard ethernet frames. Increase pool seg_len.\n", - RTE_MBUF_DEFAULT_BUF_SIZE); + "standard ethernet frames. Increase pool seg_len.\n", + RTE_MBUF_DEFAULT_BUF_SIZE); goto fail; } + if (odp_shm_info(pool_entry->shm, &shm_info)) { + ODP_ERR("Failed to query SHM info.\n"); + goto fail; + } + + page_size = shm_info.page_size; total_size = rte_mempool_calc_obj_size(elt_size, MEMPOOL_FLAGS, &sz); if (total_size != pool_entry->block_size) { ODP_ERR("DPDK pool block size not matching to ODP pool: " @@ -361,10 +359,7 @@ static struct rte_mempool *mbuf_pool_create(const char *name, goto fail; } - /* Skipped buffers have to be taken into account to populate pool - * properly. */ - mp = rte_mempool_create_empty(name, num + pool_entry->skipped_blocks, - elt_size, cache_size(num), + mp = rte_mempool_create_empty(name, num, elt_size, cache_size(num), sizeof(struct rte_pktmbuf_pool_private), rte_socket_id(), MEMPOOL_FLAGS); if (mp == NULL) { @@ -380,16 +375,34 @@ static struct rte_mempool *mbuf_pool_create(const char *name, } mbp_priv.mbuf_data_room_size = pool_entry->headroom + - pool_entry->seg_len; + pool_entry->seg_len + pool_entry->tailroom; mbp_priv.mbuf_priv_size = RTE_ALIGN(sizeof(odp_packet_hdr_t), RTE_MBUF_PRIV_ALIGN); rte_pktmbuf_pool_init(mp, &mbp_priv); - num = rte_mempool_populate_iova(mp, (char *)pool_entry->base_addr, - RTE_BAD_IOVA, pool_entry->shm_size, - NULL, NULL); - if (num <= 0) { - ODP_ERR("Failed to populate mempool: %d\n", num); + /* DPDK expects buffers that would be crossing a hugepage boundary to be aligned to the + * boundary. This isn't the case with ODP pools as boundary-crossing buffers are skipped + * and unused but still part of the pool. Thus, populate the mempool with several virtually + * and physically contiguous chunks as dictated by the skipped buffers. */ + for (uint64_t i = 0; i < pool_entry->shm_size; i += page_size) { + remainder = (page_size - offset) % total_size; + addr = pool_entry->base_addr + i + offset; + ret = rte_mempool_populate_iova(mp, (char *)addr, rte_mem_virt2iova(addr), + page_size - remainder - offset, + NULL, NULL); + + if (ret <= 0) { + ODP_ERR("Failed to populate mempool: %d\n", ret); + goto fail; + } + + populated += ret; + offset = remainder ? total_size - remainder : 0; + } + + if (populated != num) { + ODP_ERR("Failed to populate mempool with all requested blocks, populated: %u, " + "requested: %u\n", populated, num); goto fail; } @@ -556,7 +569,7 @@ static struct rte_mempool_ops odp_pool_ops = { .get_count = pool_get_count }; -MEMPOOL_REGISTER_OPS(odp_pool_ops) +RTE_MEMPOOL_REGISTER_OPS(odp_pool_ops) static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], @@ -574,9 +587,9 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, pkt_dpdk_t *pkt_dpdk = pkt_priv(pktio_entry); odp_pool_t pool = pkt_dpdk->pool; odp_pktin_config_opt_t pktin_cfg = pktio_entry->s.config.pktin; - odp_proto_layer_t parse_layer = pktio_entry->s.config.parser.layer; odp_pktio_t input = pktio_entry->s.handle; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; /* Allocate maximum sized packets */ max_len = pkt_dpdk->data_room; @@ -591,8 +604,6 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, } for (i = 0; i < num; i++) { - odp_packet_hdr_t parsed_hdr; - mbuf = mbuf_table[i]; if (odp_unlikely(mbuf->nb_segs != 1)) { ODP_ERR("Segmented buffers not supported\n"); @@ -603,31 +614,33 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, odp_prefetch(data); pkt_len = rte_pktmbuf_pkt_len(mbuf); + pkt = pkt_table[i]; + pkt_hdr = packet_hdr(pkt); - if (pktio_cls_enabled(pktio_entry)) { + if (layer) { uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; - packet_parse_reset(&parsed_hdr, 1); - packet_set_len(&parsed_hdr, pkt_len); - if (_odp_dpdk_packet_parse_common(&parsed_hdr.p, data, - pkt_len, pkt_len, - mbuf, - ODP_PROTO_LAYER_ALL, - supported_ptypes, + packet_parse_reset(pkt_hdr, 1); + if (_odp_dpdk_packet_parse_common(&pkt_hdr->p, data, + pkt_len, pkt_len, mbuf, + layer, supported_ptypes, pktin_cfg)) { odp_packet_free(pkt_table[i]); rte_pktmbuf_free(mbuf); continue; } - if (_odp_cls_classify_packet(pktio_entry, - (const uint8_t *)data, - pkt_len, pkt_len, &pool, - &parsed_hdr, false)) - goto fail; + + if (pktio_cls_enabled(pktio_entry)) { + if (_odp_cls_classify_packet(pktio_entry, + (const uint8_t *)data, + &pool, pkt_hdr)) { + odp_packet_free(pkt_table[i]); + rte_pktmbuf_free(mbuf); + continue; + } + } } - pkt = pkt_table[i]; - pkt_hdr = packet_hdr(pkt); pull_tail(pkt_hdr, max_len - pkt_len); if (frame_offset) pull_head(pkt_hdr, frame_offset); @@ -637,22 +650,7 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, pkt_hdr->input = input; - if (pktio_cls_enabled(pktio_entry)) { - _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr); - } else if (parse_layer != ODP_PROTO_LAYER_NONE) { - uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; - - if (_odp_dpdk_packet_parse_layer(pkt_hdr, mbuf, - parse_layer, - supported_ptypes, - pktin_cfg)) { - odp_packet_free(pkt); - rte_pktmbuf_free(mbuf); - continue; - } - } - - if (mbuf->ol_flags & PKT_RX_RSS_HASH) + if (mbuf->ol_flags & RTE_MBUF_F_RX_RSS_HASH) packet_set_flow_hash(pkt_hdr, mbuf->hash.rss); packet_set_ts(pkt_hdr, ts); @@ -753,12 +751,12 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg, mbuf->l2_len = pkt_p->l3_offset - pkt_p->l2_offset; if (l3_proto_v4) - mbuf->ol_flags = PKT_TX_IPV4; + mbuf->ol_flags = RTE_MBUF_F_TX_IPV4; else - mbuf->ol_flags = PKT_TX_IPV6; + mbuf->ol_flags = RTE_MBUF_F_TX_IPV6; if (ipv4_chksum_pkt) { - mbuf->ol_flags |= PKT_TX_IP_CKSUM; + mbuf->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM; ((struct rte_ipv4_hdr *)l3_hdr)->hdr_checksum = 0; mbuf->l3_len = _ODP_IPV4HDR_IHL(*(uint8_t *)l3_hdr) * 4; @@ -772,12 +770,12 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg, l4_hdr = (void *)(mbuf_data + pkt_p->l4_offset); if (udp_chksum_pkt) { - mbuf->ol_flags |= PKT_TX_UDP_CKSUM; + mbuf->ol_flags |= RTE_MBUF_F_TX_UDP_CKSUM; ((struct rte_udp_hdr *)l4_hdr)->dgram_cksum = phdr_csum(l3_proto_v4, l3_hdr, mbuf->ol_flags); } else if (tcp_chksum_pkt) { - mbuf->ol_flags |= PKT_TX_TCP_CKSUM; + mbuf->ol_flags |= RTE_MBUF_F_TX_TCP_CKSUM; ((struct rte_tcp_hdr *)l4_hdr)->cksum = phdr_csum(l3_proto_v4, l3_hdr, mbuf->ol_flags); @@ -863,9 +861,9 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, int i, nb_pkts; odp_pool_t pool; odp_pktin_config_opt_t pktin_cfg; - odp_proto_layer_t parse_layer; odp_pktio_t input; pkt_dpdk_t *pkt_dpdk; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; prefetch_pkt(mbuf_table[0]); @@ -874,15 +872,12 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, pool = pkt_dpdk->pool; set_flow_hash = pkt_dpdk->opt.set_flow_hash; pktin_cfg = pktio_entry->s.config.pktin; - parse_layer = pktio_entry->s.config.parser.layer; input = pktio_entry->s.handle; if (odp_likely(mbuf_num > 1)) prefetch_pkt(mbuf_table[1]); for (i = 0; i < mbuf_num; i++) { - odp_packet_hdr_t parsed_hdr; - if (odp_likely((i + 2) < mbuf_num)) prefetch_pkt(mbuf_table[i + 2]); @@ -895,30 +890,27 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, data = rte_pktmbuf_mtod(mbuf, char *); pkt_len = rte_pktmbuf_pkt_len(mbuf); - pkt_hdr = pkt_hdr_from_mbuf(mbuf); + packet_init(pkt_hdr, pkt_len); - if (pktio_cls_enabled(pktio_entry)) { + if (layer) { uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; - packet_parse_reset(&parsed_hdr, 1); - packet_set_len(&parsed_hdr, pkt_len); - if (_odp_dpdk_packet_parse_common(&parsed_hdr.p, data, - pkt_len, pkt_len, - mbuf, - ODP_PROTO_LAYER_ALL, - supported_ptypes, + if (_odp_dpdk_packet_parse_common(&pkt_hdr->p, data, + pkt_len, pkt_len, mbuf, + layer, supported_ptypes, pktin_cfg)) { rte_pktmbuf_free(mbuf); continue; } - if (_odp_cls_classify_packet(pktio_entry, - (const uint8_t *)data, - pkt_len, pkt_len, &pool, - &parsed_hdr, false)) { - ODP_ERR("Unable to classify packet\n"); - rte_pktmbuf_free(mbuf); - continue; + + if (pktio_cls_enabled(pktio_entry)) { + if (_odp_cls_classify_packet(pktio_entry, + (const uint8_t *)data, + &pool, pkt_hdr)) { + rte_pktmbuf_free(mbuf); + continue; + } } } @@ -926,23 +918,9 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, * are supported. */ pkt_hdr->seg_data = data; - packet_init(pkt_hdr, pkt_len); pkt_hdr->input = input; - if (pktio_cls_enabled(pktio_entry)) { - _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr); - } else if (parse_layer != ODP_PROTO_LAYER_NONE) { - uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; - - if (_odp_dpdk_packet_parse_layer(pkt_hdr, mbuf, - parse_layer, - supported_ptypes, - pktin_cfg)) { - rte_pktmbuf_free(mbuf); - continue; - } - } - if (set_flow_hash && (mbuf->ol_flags & PKT_RX_RSS_HASH)) + if (set_flow_hash && (mbuf->ol_flags & RTE_MBUF_F_RX_RSS_HASH)) packet_set_flow_hash(pkt_hdr, mbuf->hash.rss); packet_set_ts(pkt_hdr, ts); @@ -1227,11 +1205,6 @@ static int dpdk_close(pktio_entry_t *pktio_entry) rte_pktmbuf_free(pkt_dpdk->rx_cache[i].s.pkt[idx++]); } -#if RTE_VERSION < RTE_VERSION_NUM(17, 8, 0, 0) - if (pktio_entry->s.state != PKTIO_STATE_OPENED) - rte_eth_dev_close(pkt_dpdk->port_id); -#endif - return 0; } @@ -1381,18 +1354,6 @@ static void dpdk_mempool_free(struct rte_mempool *mp, void *arg ODP_UNUSED) rte_mempool_free(mp); } -/* RTE_ETH_FOREACH_DEV was introduced in v17.8, but causes a build error in - * v18.2 (only a warning, but our build system treats warnings as errors). */ -#if (RTE_VERSION >= RTE_VERSION_NUM(18, 2, 0, 0)) && \ - (RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0)) - #define ETH_FOREACH_DEV(p) \ - for (p = rte_eth_find_next(0); \ - (unsigned int)p < (unsigned int)RTE_MAX_ETHPORTS; \ - p = rte_eth_find_next(p + 1)) -#elif RTE_VERSION >= RTE_VERSION_NUM(17, 8, 0, 0) - #define ETH_FOREACH_DEV(p) RTE_ETH_FOREACH_DEV(p) -#endif - static int dpdk_pktio_term(void) { uint16_t port_id; @@ -1400,11 +1361,9 @@ static int dpdk_pktio_term(void) if (!odp_global_rw->dpdk_initialized) return 0; -#if RTE_VERSION >= RTE_VERSION_NUM(17, 8, 0, 0) - ETH_FOREACH_DEV(port_id) { + RTE_ETH_FOREACH_DEV(port_id) { rte_eth_dev_close(port_id); } -#endif if (!_ODP_DPDK_ZERO_COPY) rte_mempool_walk(dpdk_mempool_free, NULL); @@ -1574,16 +1533,18 @@ static int dpdk_init_capability(pktio_entry_t *pktio_entry, /* Check if setting MTU is supported */ ret = rte_eth_dev_set_mtu(pkt_dpdk->port_id, pkt_dpdk->mtu - _ODP_ETHHDR_LEN); - if (ret == 0) { + /* From DPDK 21.11 onwards, calling rte_eth_dev_set_mtu() before device is configured with + * rte_eth_dev_configure() will result in failure. The least hacky (unfortunately still + * very hacky) way to continue checking the support is to take into account that the + * function will fail earlier with -ENOTSUP if MTU setting is not supported by device than + * if the device was not yet configured. */ + if (ret != -ENOTSUP) { capa->set_op.op.maxlen = 1; capa->maxlen.equal = true; capa->maxlen.min_input = DPDK_MTU_MIN; capa->maxlen.max_input = pkt_dpdk->mtu_max; capa->maxlen.min_output = DPDK_MTU_MIN; capa->maxlen.max_output = pkt_dpdk->mtu_max; - } else if (ret != -ENOTSUP) { - ODP_ERR("Failed to set interface MTU: %d\n", ret); - return -1; } ptype_cnt = rte_eth_dev_get_supported_ptypes(pkt_dpdk->port_id, @@ -1669,15 +1630,6 @@ static int dpdk_init_capability(pktio_entry_t *pktio_entry, * mode change. Use system call for them. */ static void promisc_mode_check(pkt_dpdk_t *pkt_dpdk) { -#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0) - /* Enable and disable calls do not have return value */ - rte_eth_promiscuous_enable(pkt_dpdk->port_id); - - if (!rte_eth_promiscuous_get(pkt_dpdk->port_id)) - pkt_dpdk->vdev_sysc_promisc = 1; - - rte_eth_promiscuous_disable(pkt_dpdk->port_id); -#else int ret; ret = rte_eth_promiscuous_enable(pkt_dpdk->port_id); @@ -1689,7 +1641,6 @@ static void promisc_mode_check(pkt_dpdk_t *pkt_dpdk) if (ret) pkt_dpdk->vdev_sysc_promisc = 1; -#endif } static int dpdk_open(odp_pktio_t id ODP_UNUSED, @@ -1735,18 +1686,17 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED, pkt_dpdk->pool = pool; - /* rte_eth_dev_count() was removed in v18.05 */ -#if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0) - if (rte_eth_dev_count() == 0) { -#else if (rte_eth_dev_count_avail() == 0) { -#endif ODP_ERR("No DPDK ports found\n"); return -1; } memset(&dev_info, 0, sizeof(struct rte_eth_dev_info)); - rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info); + ret = rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info); + if (ret) { + ODP_ERR("Failed to read device info: %d\n", ret); + return -1; + } /* Initialize runtime options */ if (init_options(pktio_entry, &dev_info)) { @@ -1765,18 +1715,10 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED, promisc_mode_check(pkt_dpdk); -#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0) - ret = 0; - if (pkt_dpdk->opt.multicast_en) - rte_eth_allmulticast_enable(pkt_dpdk->port_id); - else - rte_eth_allmulticast_disable(pkt_dpdk->port_id); -#else if (pkt_dpdk->opt.multicast_en) ret = rte_eth_allmulticast_enable(pkt_dpdk->port_id); else ret = rte_eth_allmulticast_disable(pkt_dpdk->port_id); -#endif /* Not supported by all PMDs, so ignore the return value */ if (ret) diff --git a/platform/linux-generic/pktio/dpdk_parse.c b/platform/linux-generic/pktio/dpdk_parse.c index 0984f06c5..f28f18921 100644 --- a/platform/linux-generic/pktio/dpdk_parse.c +++ b/platform/linux-generic/pktio/dpdk_parse.c @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -26,9 +26,19 @@ #if defined(__PPC64__) && defined(vector) #undef vector #endif +#include <rte_version.h> + +#if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0) + #define RTE_MBUF_F_RX_IP_CKSUM_MASK PKT_RX_IP_CKSUM_MASK + #define RTE_MBUF_F_RX_IP_CKSUM_GOOD PKT_RX_IP_CKSUM_GOOD + #define RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN PKT_RX_IP_CKSUM_UNKNOWN + #define RTE_MBUF_F_RX_L4_CKSUM_MASK PKT_RX_L4_CKSUM_MASK + #define RTE_MBUF_F_RX_L4_CKSUM_GOOD PKT_RX_L4_CKSUM_GOOD + #define RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN PKT_RX_L4_CKSUM_UNKNOWN +#endif -#define IP4_CSUM_RESULT(ol_flags) (ol_flags & PKT_RX_IP_CKSUM_MASK) -#define L4_CSUM_RESULT(ol_flags) (ol_flags & PKT_RX_L4_CKSUM_MASK) +#define IP4_CSUM_RESULT(ol_flags) ((ol_flags) & RTE_MBUF_F_RX_IP_CKSUM_MASK) +#define L4_CSUM_RESULT(ol_flags) ((ol_flags) & RTE_MBUF_F_RX_L4_CKSUM_MASK) /** Parser helper function for Ethernet packets */ static inline uint16_t dpdk_parse_eth(packet_parser_t *prs, @@ -174,9 +184,9 @@ static inline uint8_t dpdk_parse_ipv4(packet_parser_t *prs, if (do_csum) { uint64_t packet_csum_result = IP4_CSUM_RESULT(mbuf_ol); - if (packet_csum_result == PKT_RX_IP_CKSUM_GOOD) { + if (packet_csum_result == RTE_MBUF_F_RX_IP_CKSUM_GOOD) { prs->input_flags.l3_chksum_done = 1; - } else if (packet_csum_result != PKT_RX_IP_CKSUM_UNKNOWN) { + } else if (packet_csum_result != RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN) { prs->input_flags.l3_chksum_done = 1; prs->flags.ip_err = 1; prs->flags.l3_chksum_err = 1; @@ -302,9 +312,9 @@ static inline void dpdk_parse_tcp(packet_parser_t *prs, if (do_csum) { uint64_t packet_csum_result = L4_CSUM_RESULT(mbuf_ol); - if (packet_csum_result == PKT_RX_L4_CKSUM_GOOD) { + if (packet_csum_result == RTE_MBUF_F_RX_L4_CKSUM_GOOD) { prs->input_flags.l4_chksum_done = 1; - } else if (packet_csum_result != PKT_RX_L4_CKSUM_UNKNOWN) { + } else if (packet_csum_result != RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN) { prs->input_flags.l4_chksum_done = 1; prs->flags.tcp_err = 1; prs->flags.l4_chksum_err = 1; @@ -332,9 +342,9 @@ static inline void dpdk_parse_udp(packet_parser_t *prs, if (do_csum) { uint64_t packet_csum_result = L4_CSUM_RESULT(mbuf_ol); - if (packet_csum_result == PKT_RX_L4_CKSUM_GOOD) { + if (packet_csum_result == RTE_MBUF_F_RX_L4_CKSUM_GOOD) { prs->input_flags.l4_chksum_done = 1; - } else if (packet_csum_result != PKT_RX_L4_CKSUM_UNKNOWN) { + } else if (packet_csum_result != RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN) { if (prs->input_flags.ipv4 && !udp->chksum) { prs->input_flags.l4_chksum_done = 1; } else { diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index b3232cded..3e21efecd 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -18,6 +18,7 @@ #include <odp/api/plat/packet_flag_inlines.h> #include <odp/api/plat/queue_inlines.h> +#include <odp_parse_internal.h> #include <odp_classification_internal.h> #include <odp_debug_internal.h> #include <odp_errno_define.h> @@ -26,6 +27,7 @@ #include <odp_ipsec_internal.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> +#include <odp_macros_internal.h> #include <odp_queue_if.h> #include <protocols/eth.h> @@ -158,7 +160,11 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_time_t ts_val; odp_time_t *ts = NULL; int num_rx = 0; - int failed = 0; + int packets = 0, errors = 0; + uint32_t octets = 0; + const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; if (odp_unlikely(num > QUEUE_MULTI_MAX)) num = QUEUE_MULTI_MAX; @@ -168,8 +174,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, queue = pkt_priv(pktio_entry)->loopq; nbr = odp_queue_deq_multi(queue, (odp_event_t *)hdr_tbl, num); - if (pktio_entry->s.config.pktin.bit.ts_all || - pktio_entry->s.config.pktin.bit.ts_ptp) { + if (opt.bit.ts_all || opt.bit.ts_ptp) { ts_val = odp_time_global(); ts = &ts_val; } @@ -181,61 +186,64 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pkt_len = odp_packet_len(pkt); pkt_hdr = packet_hdr(pkt); - packet_parse_reset(pkt_hdr, 1); - if (pktio_cls_enabled(pktio_entry)) { - odp_packet_t new_pkt; - odp_pool_t new_pool; + if (layer) { uint8_t *pkt_addr; - uint8_t buf[PACKET_PARSE_SEG_LEN]; + uint8_t buf[PARSE_BYTES]; int ret; uint32_t seg_len = odp_packet_seg_len(pkt); + uint64_t l4_part_sum = 0; /* Make sure there is enough data for the packet * parser in the case of a segmented packet. */ - if (odp_unlikely(seg_len < PACKET_PARSE_SEG_LEN && - pkt_len > PACKET_PARSE_SEG_LEN)) { - odp_packet_copy_to_mem(pkt, 0, - PACKET_PARSE_SEG_LEN, - buf); - seg_len = PACKET_PARSE_SEG_LEN; + if (odp_unlikely(seg_len < PARSE_BYTES && + pkt_len > seg_len)) { + seg_len = MIN(pkt_len, PARSE_BYTES); + odp_packet_copy_to_mem(pkt, 0, seg_len, buf); pkt_addr = buf; } else { pkt_addr = odp_packet_data(pkt); } - ret = _odp_cls_classify_packet(pktio_entry, pkt_addr, - pkt_len, seg_len, - &new_pool, pkt_hdr, true); - if (ret) { - failed++; + packet_parse_reset(pkt_hdr, 1); + ret = _odp_packet_parse_common(&pkt_hdr->p, pkt_addr, pkt_len, + seg_len, layer, chksums, + &l4_part_sum, opt); + if (ret) + errors++; + + if (ret < 0) { odp_packet_free(pkt); continue; } - if (new_pool != odp_packet_pool(pkt)) { - new_pkt = odp_packet_copy(pkt, new_pool); - - odp_packet_free(pkt); + if (pktio_cls_enabled(pktio_entry)) { + odp_packet_t new_pkt; + odp_pool_t new_pool; - if (new_pkt == ODP_PACKET_INVALID) { - failed++; + ret = _odp_cls_classify_packet(pktio_entry, pkt_addr, + &new_pool, pkt_hdr); + if (ret) { + odp_packet_free(pkt); continue; } - pkt = new_pkt; - pkt_hdr = packet_hdr(new_pkt); + if (new_pool != odp_packet_pool(pkt)) { + new_pkt = odp_packet_copy(pkt, new_pool); + + odp_packet_free(pkt); + + if (new_pkt == ODP_PACKET_INVALID) { + pktio_entry->s.stats.in_discards++; + continue; + } + + pkt = new_pkt; + pkt_hdr = packet_hdr(new_pkt); + } } - } else { - odp_packet_parse_param_t param; - - /* - * Use odp_packet_parse() which can handle segmented - * packets. - */ - param.proto = ODP_PROTO_ETH; - param.last_layer = pktio_entry->s.config.parser.layer; - param.chksums = pktio_entry->s.in_chksums; - odp_packet_parse(packet_handle(pkt_hdr), 0, ¶m); + + if (layer >= ODP_PROTO_LAYER_L4) + _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); } packet_set_ts(pkt_hdr, ts); @@ -247,12 +255,17 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_packet_has_ipsec(pkt)) _odp_ipsec_try_inline(&pkt); - pktio_entry->s.stats.in_octets += pkt_len; + if (!pkt_hdr->p.flags.all.error) { + octets += pkt_len; + packets++; + } + pkts[num_rx++] = pkt; } - pktio_entry->s.stats.in_errors += failed; - pktio_entry->s.stats.in_packets += num_rx - failed; + pktio_entry->s.stats.in_octets += octets; + pktio_entry->s.stats.in_packets += packets; + pktio_entry->s.stats.in_errors += errors; odp_ticketlock_unlock(&pktio_entry->s.rxl); @@ -533,6 +546,7 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry) capa->stats.pktio.counter.in_octets = 1; capa->stats.pktio.counter.in_packets = 1; capa->stats.pktio.counter.in_errors = 1; + capa->stats.pktio.counter.in_discards = 1; capa->stats.pktio.counter.out_octets = 1; capa->stats.pktio.counter.out_packets = 1; capa->stats.pktin_queue.counter.octets = 1; diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index 0b1345e50..94b88e21e 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -16,6 +16,7 @@ #include <odp/api/time.h> #include <odp/api/plat/time_inlines.h> +#include <odp_parse_internal.h> #include <odp_packet_io_internal.h> #include <odp_packet_io_stats.h> #include <odp_ethtool_stats.h> @@ -822,11 +823,14 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, odp_packet_t pkt; odp_pool_t pool = pkt_priv(pktio_entry)->pool; odp_packet_hdr_t *pkt_hdr; - odp_packet_hdr_t parsed_hdr; int i; int num; uint32_t max_len; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; + int num_rx = 0; + const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; /* Allocate maximum sized packets */ max_len = pkt_priv(pktio_entry)->mtu; @@ -837,45 +841,50 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, for (i = 0; i < num; i++) { netmap_slot_t slot; uint16_t len; + const uint8_t *buf; + uint64_t l4_part_sum = 0; slot = slot_tbl[i]; len = slot.len; + buf = (const uint8_t *)slot.buf; odp_prefetch(slot.buf); - if (pktio_cls_enabled(pktio_entry)) { - if (_odp_cls_classify_packet(pktio_entry, - (const uint8_t *)slot.buf, len, - len, &pool, &parsed_hdr, true)) - goto fail; + pkt = pkt_tbl[num_rx]; + pkt_hdr = packet_hdr(pkt); + + if (layer) { + if (_odp_packet_parse_common(&pkt_hdr->p, buf, len, len, + layer, chksums, &l4_part_sum, opt) < 0) + continue; + + if (pktio_cls_enabled(pktio_entry)) { + if (_odp_cls_classify_packet(pktio_entry, buf, &pool, + pkt_hdr)) + continue; + } } - pkt = pkt_tbl[i]; - pkt_hdr = packet_hdr(pkt); pull_tail(pkt_hdr, max_len - len); if (frame_offset) pull_head(pkt_hdr, frame_offset); if (odp_packet_copy_from_mem(pkt, 0, len, slot.buf) != 0) - goto fail; + break; pkt_hdr->input = pktio_entry->s.handle; - if (pktio_cls_enabled(pktio_entry)) - _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr); - else - _odp_packet_parse_layer(pkt_hdr, - pktio_entry->s.config.parser.layer, - pktio_entry->s.in_chksums); + if (layer >= ODP_PROTO_LAYER_L4) + _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); packet_set_ts(pkt_hdr, ts); + num_rx++; } - return i; + if (num_rx < num) + odp_packet_free_multi(&pkt_tbl[num_rx], num - num_rx); -fail: - odp_packet_free_multi(&pkt_tbl[i], num - i); - return i; + return num_rx; } static inline int netmap_recv_desc(pktio_entry_t *pktio_entry, @@ -1042,11 +1051,11 @@ static int netmap_recv_tmo(pktio_entry_t *pktio_entry, int index, } static int netmap_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[], - int num_q, odp_packet_t pkt_table[], int num, - unsigned *from, uint64_t usecs) + uint32_t num_q, odp_packet_t pkt_table[], int num, + uint32_t *from, uint64_t usecs) { struct timeval timeout; - int i; + uint32_t i; int ret; int maxfd = -1, maxfd2; fd_set readfds; diff --git a/platform/linux-generic/pktio/null.c b/platform/linux-generic/pktio/null.c index 90e113ec6..a6498da8c 100644 --- a/platform/linux-generic/pktio/null.c +++ b/platform/linux-generic/pktio/null.c @@ -71,9 +71,9 @@ static int null_recv_tmo(pktio_entry_t *pktio_entry ODP_UNUSED, } static int null_recv_mq_tmo(pktio_entry_t *pktio_entry[] ODP_UNUSED, - int index[] ODP_UNUSED, int num_q ODP_UNUSED, + int index[] ODP_UNUSED, uint32_t num_q ODP_UNUSED, odp_packet_t pkt_table[] ODP_UNUSED, - int num ODP_UNUSED, unsigned *from ODP_UNUSED, + int num ODP_UNUSED, uint32_t *from ODP_UNUSED, uint64_t usecs) { struct timeval timeout; diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c index 7290626c2..af94ffa72 100644 --- a/platform/linux-generic/pktio/pcap.c +++ b/platform/linux-generic/pktio/pcap.c @@ -46,6 +46,7 @@ #include <odp/api/plat/packet_inlines.h> +#include <odp_parse_internal.h> #include <odp_classification_internal.h> #include <odp_debug_internal.h> #include <odp_global_data.h> @@ -248,7 +249,12 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pkt_pcap_t *pcap = pkt_priv(pktio_entry); odp_time_t ts_val; odp_time_t *ts = NULL; + int packets = 0, errors = 0; + uint32_t octets = 0; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; + const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; odp_ticketlock_lock(&pktio_entry->s.rxl); @@ -256,8 +262,7 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_ticketlock_unlock(&pktio_entry->s.rxl); return 0; } - if (pktio_entry->s.config.pktin.bit.ts_all || - pktio_entry->s.config.pktin.bit.ts_ptp) + if (opt.bit.ts_all || opt.bit.ts_ptp) ts = &ts_val; for (i = 0; i < num; ) { @@ -291,42 +296,64 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, break; } - if (pktio_cls_enabled(pktio_entry)) { - odp_packet_t new_pkt; + if (layer) { + uint64_t l4_part_sum = 0; - ret = _odp_cls_classify_packet(pktio_entry, data, - pkt_len, pkt_len, - &new_pool, pkt_hdr, true); - if (ret) { + ret = _odp_packet_parse_common(&pkt_hdr->p, data, pkt_len, + pkt_len, layer, chksums, + &l4_part_sum, opt); + if (ret) + errors++; + + if (ret < 0) { odp_packet_free(pkt); continue; } - if (new_pool != pcap->pool) { - new_pkt = odp_packet_copy(pkt, new_pool); - odp_packet_free(pkt); + if (pktio_cls_enabled(pktio_entry)) { + odp_packet_t new_pkt; - if (odp_unlikely(new_pkt == ODP_PACKET_INVALID)) + ret = _odp_cls_classify_packet(pktio_entry, data, + &new_pool, pkt_hdr); + if (ret) { + odp_packet_free(pkt); continue; + } + if (new_pool != pcap->pool) { + new_pkt = odp_packet_copy(pkt, new_pool); + + odp_packet_free(pkt); + + if (odp_unlikely(new_pkt == ODP_PACKET_INVALID)) { + pktio_entry->s.stats.in_discards++; + continue; + } - pkt = new_pkt; - pkt_hdr = packet_hdr(new_pkt); + pkt = new_pkt; + pkt_hdr = packet_hdr(new_pkt); + } } - } else { - _odp_packet_parse_layer(pkt_hdr, - pktio_entry->s.config.parser.layer, - pktio_entry->s.in_chksums); + + if (layer >= ODP_PROTO_LAYER_L4) + _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); } - pktio_entry->s.stats.in_octets += pkt_hdr->frame_len; packet_set_ts(pkt_hdr, ts); pkt_hdr->input = pktio_entry->s.handle; + if (!pkt_hdr->p.flags.all.error) { + octets += pkt_len; + packets++; + } + pkts[i] = pkt; i++; } - pktio_entry->s.stats.in_packets += i; + + pktio_entry->s.stats.in_octets += octets; + pktio_entry->s.stats.in_packets += packets; + pktio_entry->s.stats.in_errors += errors; odp_ticketlock_unlock(&pktio_entry->s.rxl); @@ -441,6 +468,8 @@ static int pcapif_capability(pktio_entry_t *pktio_entry ODP_UNUSED, capa->stats.pktio.counter.in_octets = 1; capa->stats.pktio.counter.in_packets = 1; + capa->stats.pktio.counter.in_discards = 1; + capa->stats.pktio.counter.in_errors = 1; capa->stats.pktio.counter.out_octets = 1; capa->stats.pktio.counter.out_packets = 1; diff --git a/platform/linux-generic/pktio/pktio_common.c b/platform/linux-generic/pktio/pktio_common.c index d2f4d7219..1f3cf951e 100644 --- a/platform/linux-generic/pktio/pktio_common.c +++ b/platform/linux-generic/pktio/pktio_common.c @@ -11,13 +11,13 @@ static int sock_recv_mq_tmo_select(pktio_entry_t * const *entry, const int index[], - unsigned int num_q, unsigned int *from, + uint32_t num_q, uint32_t *from, odp_packet_t packets[], int num, uint64_t usecs, fd_set *readfds, int maxfd) { struct timeval timeout; - unsigned int i; + uint32_t i; int ret; for (i = 0; i < num_q; i++) { @@ -50,17 +50,17 @@ static int sock_recv_mq_tmo_select(pktio_entry_t * const *entry, } int _odp_sock_recv_mq_tmo_try_int_driven(const struct odp_pktin_queue_t queues[], - unsigned int num_q, unsigned int *from, + uint32_t num_q, uint32_t *from, odp_packet_t packets[], int num, uint64_t usecs, int *trial_successful) { - unsigned int i; + uint32_t i; pktio_entry_t *entry[num_q]; int index[num_q]; fd_set readfds; int maxfd = -1; - int (*impl)(pktio_entry_t *entry[], int index[], int num_q, - odp_packet_t packets[], int num, unsigned int *from, + int (*impl)(pktio_entry_t *entry[], int index[], uint32_t num_q, + odp_packet_t packets[], int num, uint32_t *from, uint64_t wait_usecs) = NULL; int impl_set = 0; diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c index 81c178b9d..9d1bbe545 100644 --- a/platform/linux-generic/pktio/socket.c +++ b/platform/linux-generic/pktio/socket.c @@ -15,12 +15,14 @@ #include <odp/api/ticketlock.h> #include <odp_socket_common.h> +#include <odp_parse_internal.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> #include <odp_packet_io_stats.h> #include <odp_debug_internal.h> #include <odp_errno_define.h> #include <odp_classification_internal.h> +#include <odp_macros_internal.h> #include <sys/socket.h> #include <stdio.h> @@ -233,6 +235,9 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, int i; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; uint32_t alloc_len = pkt_sock->mtu + frame_offset; + const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; memset(msgvec, 0, sizeof(msgvec)); @@ -249,8 +254,7 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, recv_msgs = recvmmsg(sockfd, msgvec, nb_pkts, MSG_DONTWAIT, NULL); odp_ticketlock_unlock(&pkt_sock->rx_lock); - if (pktio_entry->s.config.pktin.bit.ts_all || - pktio_entry->s.config.pktin.bit.ts_ptp) { + if (opt.bit.ts_all || opt.bit.ts_ptp) { ts_val = odp_time_global(); ts = &ts_val; } @@ -262,25 +266,48 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint16_t pkt_len = msgvec[i].msg_len; int ret; + uint64_t l4_part_sum = 0; if (odp_unlikely(msgvec[i].msg_hdr.msg_flags & MSG_TRUNC)) { odp_packet_free(pkt); ODP_DBG("dropped truncated packet\n"); continue; } - if (pktio_cls_enabled(pktio_entry)) { - uint16_t seg_len = pkt_len; - if (msgvec[i].msg_hdr.msg_iov->iov_len < pkt_len) - seg_len = msgvec[i].msg_hdr.msg_iov->iov_len; + ret = odp_packet_trunc_tail(&pkt, odp_packet_len(pkt) - pkt_len, + NULL, NULL); + if (ret < 0) { + ODP_ERR("trunc_tail failed"); + odp_packet_free(pkt); + continue; + } + + if (layer) { + uint8_t buf[PARSE_BYTES]; + uint16_t seg_len = msgvec[i].msg_hdr.msg_iov->iov_len; - if (_odp_cls_classify_packet(pktio_entry, base, pkt_len, - seg_len, &pool, pkt_hdr, - true)) { - ODP_ERR("_odp_cls_classify_packet failed"); + /* Make sure there is enough data for the packet + * parser in the case of a segmented packet. */ + if (odp_unlikely(seg_len < PARSE_BYTES && pkt_len > seg_len)) { + seg_len = MIN(pkt_len, PARSE_BYTES); + odp_packet_copy_to_mem(pkt, 0, seg_len, buf); + base = buf; + } + + if (_odp_packet_parse_common(&pkt_hdr->p, base, pkt_len, + seg_len, layer, chksums, + &l4_part_sum, opt) < 0) { odp_packet_free(pkt); continue; } + + if (pktio_cls_enabled(pktio_entry)) { + if (_odp_cls_classify_packet(pktio_entry, base, &pool, + pkt_hdr)) { + odp_packet_free(pkt); + continue; + } + } } /* Don't receive packets sent by ourselves */ @@ -290,20 +317,10 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, continue; } - ret = odp_packet_trunc_tail(&pkt, odp_packet_len(pkt) - pkt_len, - NULL, NULL); - if (ret < 0) { - ODP_ERR("trunk_tail failed"); - odp_packet_free(pkt); - continue; - } - pkt_hdr->input = pktio_entry->s.handle; - if (!pktio_cls_enabled(pktio_entry)) - _odp_packet_parse_layer(pkt_hdr, - pktio_entry->s.config.parser.layer, - pktio_entry->s.in_chksums); + if (layer >= ODP_PROTO_LAYER_L4) + _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); packet_set_ts(pkt_hdr, ts); @@ -359,11 +376,11 @@ static int sock_recv_tmo(pktio_entry_t *pktio_entry, int index, } static int sock_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[], - int num_q, odp_packet_t pkt_table[], int num, - unsigned *from, uint64_t usecs) + uint32_t num_q, odp_packet_t pkt_table[], int num, + uint32_t *from, uint64_t usecs) { struct timeval timeout; - int i; + uint32_t i; int ret; int maxfd = -1, maxfd2; fd_set readfds; diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index f84834610..7824b0e91 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -16,6 +16,7 @@ #include <odp/api/plat/packet_inlines.h> #include <odp_socket_common.h> +#include <odp_parse_internal.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> #include <odp_packet_io_stats.h> @@ -150,9 +151,11 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, odp_pool_t pool = pkt_sock->pool; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; uint16_t vlan_len = 0; + const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; - if (pktio_entry->s.config.pktin.bit.ts_all || - pktio_entry->s.config.pktin.bit.ts_ptp) + if (opt.bit.ts_all || opt.bit.ts_ptp) ts = &ts_val; ring = &pkt_sock->rx_ring; @@ -163,8 +166,8 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, struct tpacket2_hdr *tp_hdr; odp_packet_t pkt; odp_packet_hdr_t *hdr; - odp_packet_hdr_t parsed_hdr; int ret; + uint64_t l4_part_sum = 0; tp_hdr = (void *)next_ptr; @@ -212,18 +215,29 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, continue; } - if (pktio_cls_enabled(pktio_entry)) { - if (_odp_cls_classify_packet(pktio_entry, pkt_buf, pkt_len, - pkt_len, &pool, &parsed_hdr, - true)) { + hdr = packet_hdr(pkt); + + if (layer) { + if (_odp_packet_parse_common(&hdr->p, pkt_buf, pkt_len, + pkt_len, layer, chksums, + &l4_part_sum, opt) < 0) { odp_packet_free(pkt); tp_hdr->tp_status = TP_STATUS_KERNEL; frame_num = next_frame_num; continue; } + + if (pktio_cls_enabled(pktio_entry)) { + if (_odp_cls_classify_packet(pktio_entry, pkt_buf, + &pool, hdr)) { + odp_packet_free(pkt); + tp_hdr->tp_status = TP_STATUS_KERNEL; + frame_num = next_frame_num; + continue; + } + } } - hdr = packet_hdr(pkt); if (frame_offset) pull_head(hdr, frame_offset); @@ -276,12 +290,8 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, hdr->input = pktio_entry->s.handle; - if (pktio_cls_enabled(pktio_entry)) - _odp_packet_copy_cls_md(hdr, &parsed_hdr); - else - _odp_packet_parse_layer(hdr, - pktio_entry->s.config.parser.layer, - pktio_entry->s.in_chksums); + if (layer >= ODP_PROTO_LAYER_L4) + _odp_packet_l4_chksum(hdr, chksums, l4_part_sum); packet_set_ts(hdr, ts); @@ -727,11 +737,11 @@ static int sock_mmap_recv_tmo(pktio_entry_t *pktio_entry, int index, } static int sock_mmap_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[], - int num_q, odp_packet_t pkt_table[], int num, - unsigned *from, uint64_t usecs) + uint32_t num_q, odp_packet_t pkt_table[], int num, + uint32_t *from, uint64_t usecs) { struct timeval timeout; - int i; + uint32_t i; int ret; int maxfd = -1, maxfd2; fd_set readfds; diff --git a/platform/linux-generic/pktio/tap.c b/platform/linux-generic/pktio/tap.c index d298c8982..40c46b43c 100644 --- a/platform/linux-generic/pktio/tap.c +++ b/platform/linux-generic/pktio/tap.c @@ -38,6 +38,7 @@ #include <odp/api/plat/packet_inlines.h> +#include <odp_parse_internal.h> #include <odp_debug_internal.h> #include <odp_socket_common.h> #include <odp_packet_internal.h> @@ -286,14 +287,27 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data, odp_packet_hdr_t *pkt_hdr; odp_packet_hdr_t parsed_hdr; int num; + uint64_t l4_part_sum = 0; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; - - if (pktio_cls_enabled(pktio_entry)) { - if (_odp_cls_classify_packet(pktio_entry, data, len, len, - &pkt_priv(pktio_entry)->pool, - &parsed_hdr, true)) { + const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; + + if (layer) { + packet_parse_reset(&parsed_hdr, 1); + packet_set_len(&parsed_hdr, len); + if (_odp_packet_parse_common(&parsed_hdr.p, data, len, len, layer, + chksums, &l4_part_sum, opt) < 0) { return ODP_PACKET_INVALID; } + + if (pktio_cls_enabled(pktio_entry)) { + if (_odp_cls_classify_packet(pktio_entry, data, + &pkt_priv(pktio_entry)->pool, + &parsed_hdr)) { + return ODP_PACKET_INVALID; + } + } } num = _odp_packet_alloc_multi(pkt_priv(pktio_entry)->pool, @@ -312,12 +326,12 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data, return ODP_PACKET_INVALID; } - if (pktio_cls_enabled(pktio_entry)) + if (layer) { _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr); - else - _odp_packet_parse_layer(pkt_hdr, - pktio_entry->s.config.parser.layer, - pktio_entry->s.in_chksums); + + if (layer >= ODP_PROTO_LAYER_L4) + _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); + } packet_set_ts(pkt_hdr, ts); pkt_hdr->input = pktio_entry->s.handle; @@ -335,6 +349,7 @@ static int tap_pktio_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, uint8_t buf[mtu]; odp_time_t ts_val; odp_time_t *ts = NULL; + int num_rx = 0; odp_ticketlock_lock(&pktio_entry->s.rxl); @@ -355,14 +370,15 @@ static int tap_pktio_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, break; } - pkts[i] = pack_odp_pkt(pktio_entry, buf, retval, ts); - if (pkts[i] == ODP_PACKET_INVALID) + pkts[num_rx] = pack_odp_pkt(pktio_entry, buf, retval, ts); + if (pkts[num_rx] == ODP_PACKET_INVALID) break; + num_rx++; } odp_ticketlock_unlock(&pktio_entry->s.rxl); - return i; + return num_rx; } static int tap_pktio_send_lockless(pktio_entry_t *pktio_entry, |