aboutsummaryrefslogtreecommitdiff
path: root/platform
diff options
context:
space:
mode:
authorMatias Elo <matias.elo@nokia.com>2022-05-04 15:40:33 +0300
committerMatias Elo <matias.elo@nokia.com>2022-05-04 15:40:33 +0300
commit2f0df4cb4acdee52f436cad8476ccf662821b7b5 (patch)
tree085136f3d98d8187b17ea584c992c8117bdedd8f /platform
parent2f869d2698349bf75ec2404d4ab7d7f3b5b4c88b (diff)
parente361227ac00851e8720871e927b015c4c0e6a895 (diff)
Merge tag 'v1.36.0.0' of https://github.com/OpenDataPlane/odp into odp-dpdk
Signed-off-by: Matias Elo <matias.elo@nokia.com>
Diffstat (limited to 'platform')
-rw-r--r--platform/linux-generic/Makefile.am12
-rw-r--r--platform/linux-generic/arch/aarch64/odp_atomic.h95
-rw-r--r--platform/linux-generic/arch/aarch64/odp_crypto_armv8.c265
-rw-r--r--platform/linux-generic/arch/aarch64/odp_llsc.h8
-rw-r--r--platform/linux-generic/arch/arm/odp_atomic.h3
-rw-r--r--platform/linux-generic/arch/default/odp_atomic.h14
-rw-r--r--platform/linux-generic/include-abi/odp/api/abi/event.h34
-rw-r--r--platform/linux-generic/include-abi/odp/api/abi/event_types.h61
-rw-r--r--platform/linux-generic/include-abi/odp/api/abi/timer.h8
-rw-r--r--platform/linux-generic/include/odp/api/plat/buffer_inlines.h2
-rw-r--r--platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h195
-rw-r--r--platform/linux-generic/include/odp/api/plat/packet_inline_types.h7
-rw-r--r--platform/linux-generic/include/odp/api/plat/packet_inlines.h32
-rw-r--r--platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h2
-rw-r--r--platform/linux-generic/include/odp/api/plat/queue_inline_types.h1
-rw-r--r--platform/linux-generic/include/odp/api/plat/time_inlines.h12
-rw-r--r--platform/linux-generic/include/odp/api/plat/timer_inline_types.h37
-rw-r--r--platform/linux-generic/include/odp/api/plat/timer_inlines.h78
-rw-r--r--platform/linux-generic/include/odp_atomic_internal.h13
-rw-r--r--platform/linux-generic/include/odp_classification_datamodel.h4
-rw-r--r--platform/linux-generic/include/odp_classification_internal.h7
-rw-r--r--platform/linux-generic/include/odp_llqueue.h3
-rw-r--r--platform/linux-generic/include/odp_macros_internal.h2
-rw-r--r--platform/linux-generic/include/odp_packet_dpdk.h14
-rw-r--r--platform/linux-generic/include/odp_packet_internal.h48
-rw-r--r--platform/linux-generic/include/odp_packet_io_internal.h18
-rw-r--r--platform/linux-generic/include/odp_parse_internal.h112
-rw-r--r--platform/linux-generic/include/odp_schedule_if.h2
-rw-r--r--platform/linux-generic/include/odp_types_internal.h24
-rw-r--r--platform/linux-generic/odp_classification.c119
-rw-r--r--platform/linux-generic/odp_crypto_null.c7
-rw-r--r--platform/linux-generic/odp_crypto_openssl.c10
-rw-r--r--platform/linux-generic/odp_dma.c1
-rw-r--r--platform/linux-generic/odp_event.c5
-rw-r--r--platform/linux-generic/odp_ipsec.c7
-rw-r--r--platform/linux-generic/odp_ipsec_sad.c5
-rw-r--r--platform/linux-generic/odp_packet.c568
-rw-r--r--platform/linux-generic/odp_packet_flags.c138
-rw-r--r--platform/linux-generic/odp_packet_io.c122
-rw-r--r--platform/linux-generic/odp_parse.c471
-rw-r--r--platform/linux-generic/odp_pool.c13
-rw-r--r--platform/linux-generic/odp_queue_lf.c17
-rw-r--r--platform/linux-generic/odp_schedule_scalable.c16
-rw-r--r--platform/linux-generic/odp_schedule_sp.c1
-rw-r--r--platform/linux-generic/odp_time.c9
-rw-r--r--platform/linux-generic/odp_timer.c127
-rw-r--r--platform/linux-generic/odp_timer_api.c11
-rw-r--r--platform/linux-generic/odp_traffic_mngr.c123
-rw-r--r--platform/linux-generic/pktio/dpdk.c254
-rw-r--r--platform/linux-generic/pktio/dpdk_parse.c28
-rw-r--r--platform/linux-generic/pktio/loop.c96
-rw-r--r--platform/linux-generic/pktio/netmap.c53
-rw-r--r--platform/linux-generic/pktio/null.c4
-rw-r--r--platform/linux-generic/pktio/pcap.c69
-rw-r--r--platform/linux-generic/pktio/pktio_common.c12
-rw-r--r--platform/linux-generic/pktio/socket.c67
-rw-r--r--platform/linux-generic/pktio/socket_mmap.c44
-rw-r--r--platform/linux-generic/pktio/tap.c42
58 files changed, 1981 insertions, 1571 deletions
diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am
index 47e5cf759..d4e94f677 100644
--- a/platform/linux-generic/Makefile.am
+++ b/platform/linux-generic/Makefile.am
@@ -49,7 +49,9 @@ odpapiplatinclude_HEADERS = \
include/odp/api/plat/sync_inlines.h \
include/odp/api/plat/thread_inlines.h \
include/odp/api/plat/ticketlock_inlines.h \
- include/odp/api/plat/time_inlines.h
+ include/odp/api/plat/time_inlines.h \
+ include/odp/api/plat/timer_inlines.h \
+ include/odp/api/plat/timer_inline_types.h
odpapiabiarchinclude_HEADERS += \
include-abi/odp/api/abi/align.h \
@@ -65,6 +67,7 @@ odpapiabiarchinclude_HEADERS += \
include-abi/odp/api/abi/dma_types.h \
include-abi/odp/api/abi/errno.h \
include-abi/odp/api/abi/event.h \
+ include-abi/odp/api/abi/event_types.h \
include-abi/odp/api/abi/hash.h \
include-abi/odp/api/abi/init.h \
include-abi/odp/api/abi/ipsec.h \
@@ -95,6 +98,7 @@ odpapiabiarchinclude_HEADERS += \
include-abi/odp/api/abi/thrmask.h \
include-abi/odp/api/abi/ticketlock.h \
include-abi/odp/api/abi/time.h \
+ include-abi/odp/api/abi/timer.h \
include-abi/odp/api/abi/timer_types.h \
include-abi/odp/api/abi/traffic_mngr.h \
include-abi/odp/api/abi/version.h
@@ -126,6 +130,7 @@ noinst_HEADERS = \
include/odp_packet_dpdk.h \
include/odp_packet_internal.h \
include/odp_packet_io_internal.h \
+ include/odp_parse_internal.h \
include/odp_socket_common.h \
include/odp_packet_io_stats_common.h \
include/odp_packet_io_stats.h \
@@ -160,6 +165,7 @@ noinst_HEADERS = \
include/odp_timer_internal.h \
include/odp_timer_wheel_internal.h \
include/odp_traffic_mngr_internal.h \
+ include/odp_types_internal.h \
include/odp_event_vector_internal.h \
include/protocols/eth.h \
include/protocols/ip.h \
@@ -201,6 +207,7 @@ __LIB__libodp_linux_la_SOURCES = \
odp_packet_vector.c \
odp_packet_flags.c \
odp_packet_io.c \
+ odp_parse.c \
odp_pkt_queue.c \
odp_pool.c \
odp_queue_basic.c \
@@ -279,7 +286,8 @@ __LIB__libodp_linux_la_SOURCES += \
odp_sync_api.c \
odp_thread_api.c \
odp_ticketlock_api.c \
- odp_time_api.c
+ odp_time_api.c \
+ odp_timer_api.c
endif
if ARCH_IS_ARM
diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.h b/platform/linux-generic/arch/aarch64/odp_atomic.h
index 445d91e28..d3b8ea4dc 100644
--- a/platform/linux-generic/arch/aarch64/odp_atomic.h
+++ b/platform/linux-generic/arch/aarch64/odp_atomic.h
@@ -12,6 +12,7 @@
#error This file should not be included directly, please include odp_cpu.h
#endif
+#include <odp_types_internal.h>
#include <limits.h>
#ifdef CONFIG_DMBSTR
@@ -36,22 +37,18 @@ do { \
#define LL_MO(mo) (HAS_ACQ((mo)) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED)
#define SC_MO(mo) (HAS_RLS((mo)) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED)
-/* Prevent warnings about ISO C not supporting __int128 */
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wpedantic"
-
#ifndef __ARM_FEATURE_QRDMX /* Feature only available in v8.1a and beyond */
static inline bool
-__lockfree_compare_exchange_16(register __int128 *var, __int128 *exp,
- register __int128 neu, bool weak, int mo_success,
+__lockfree_compare_exchange_16(register _odp_u128_t *var, _odp_u128_t *exp,
+ register _odp_u128_t neu, bool weak, int mo_success,
int mo_failure)
{
(void)weak; /* Always do strong CAS or we can't perform atomic read */
/* Ignore memory ordering for failure, memory order for
* success must be stronger or equal. */
(void)mo_failure;
- register __int128 old;
- register __int128 expected;
+ register _odp_u128_t old;
+ register _odp_u128_t expected;
int ll_mo = LL_MO(mo_success);
int sc_mo = SC_MO(mo_success);
@@ -66,10 +63,10 @@ __lockfree_compare_exchange_16(register __int128 *var, __int128 *exp,
return old == expected;
}
-static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu,
- int mo)
+static inline _odp_u128_t __lockfree_exchange_16(_odp_u128_t *var,
+ _odp_u128_t neu, int mo)
{
- register __int128 old;
+ register _odp_u128_t old;
int ll_mo = LL_MO(mo);
int sc_mo = SC_MO(mo);
@@ -81,10 +78,10 @@ static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu,
return old;
}
-static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask,
- int mo)
+static inline _odp_u128_t __lockfree_fetch_and_16(_odp_u128_t *var,
+ _odp_u128_t mask, int mo)
{
- register __int128 old;
+ register _odp_u128_t old;
int ll_mo = LL_MO(mo);
int sc_mo = SC_MO(mo);
@@ -96,10 +93,10 @@ static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask,
return old;
}
-static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask,
- int mo)
+static inline _odp_u128_t __lockfree_fetch_or_16(_odp_u128_t *var,
+ _odp_u128_t mask, int mo)
{
- register __int128 old;
+ register _odp_u128_t old;
int ll_mo = LL_MO(mo);
int sc_mo = SC_MO(mo);
@@ -113,8 +110,8 @@ static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask,
#else
-static inline __int128_t cas_u128(__int128_t *ptr, __int128_t old_val,
- __int128_t new_val, int mo)
+static inline _odp_u128_t cas_u128(_odp_u128_t *ptr, _odp_u128_t old_val,
+ _odp_u128_t new_val, int mo)
{
/* CASP instructions require that the first register number is paired */
register uint64_t old0 __asm__ ("x0");
@@ -151,18 +148,18 @@ static inline __int128_t cas_u128(__int128_t *ptr, __int128_t old_val,
abort();
}
- return ((__int128)old0) | (((__int128)old1) << 64);
+ return ((_odp_u128_t)old0) | (((_odp_u128_t)old1) << 64);
}
static inline bool
-__lockfree_compare_exchange_16(register __int128 *var, __int128 *exp,
- register __int128 neu, bool weak, int mo_success,
+__lockfree_compare_exchange_16(register _odp_u128_t *var, _odp_u128_t *exp,
+ register _odp_u128_t neu, bool weak, int mo_success,
int mo_failure)
{
(void)weak;
(void)mo_failure;
- __int128 old;
- __int128 expected;
+ _odp_u128_t old;
+ _odp_u128_t expected;
expected = *exp;
old = cas_u128(var, expected, neu, mo_success);
@@ -170,11 +167,11 @@ __lockfree_compare_exchange_16(register __int128 *var, __int128 *exp,
return old == expected;
}
-static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu,
- int mo)
+static inline _odp_u128_t __lockfree_exchange_16(_odp_u128_t *var,
+ _odp_u128_t neu, int mo)
{
- __int128 old;
- __int128 expected;
+ _odp_u128_t old;
+ _odp_u128_t expected;
do {
expected = *var;
@@ -183,11 +180,11 @@ static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu,
return old;
}
-static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask,
- int mo)
+static inline _odp_u128_t __lockfree_fetch_and_16(_odp_u128_t *var,
+ _odp_u128_t mask, int mo)
{
- __int128 old;
- __int128 expected;
+ _odp_u128_t old;
+ _odp_u128_t expected;
do {
expected = *var;
@@ -196,11 +193,11 @@ static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask,
return old;
}
-static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask,
- int mo)
+static inline _odp_u128_t __lockfree_fetch_or_16(_odp_u128_t *var,
+ _odp_u128_t mask, int mo)
{
- __int128 old;
- __int128 expected;
+ _odp_u128_t old;
+ _odp_u128_t expected;
do {
expected = *var;
@@ -211,9 +208,9 @@ static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask,
#endif /* __ARM_FEATURE_QRDMX */
-static inline __int128 __lockfree_load_16(__int128 *var, int mo)
+static inline _odp_u128_t __lockfree_load_16(_odp_u128_t *var, int mo)
{
- __int128 old = *var; /* Possibly torn read */
+ _odp_u128_t old = *var; /* Possibly torn read */
/* Do CAS to ensure atomicity
* Either CAS succeeds (writing back the same value)
@@ -223,19 +220,17 @@ static inline __int128 __lockfree_load_16(__int128 *var, int mo)
return old;
}
-__extension__ typedef unsigned __int128 _u128_t;
-
-static inline _u128_t lockfree_load_u128(_u128_t *atomic)
+static inline _odp_u128_t lockfree_load_u128(_odp_u128_t *atomic)
{
- return __lockfree_load_16((__int128 *)atomic, __ATOMIC_RELAXED);
+ return __lockfree_load_16((_odp_u128_t *)atomic, __ATOMIC_RELAXED);
}
-static inline int lockfree_cas_acq_rel_u128(_u128_t *atomic,
- _u128_t old_val,
- _u128_t new_val)
+static inline int lockfree_cas_acq_rel_u128(_odp_u128_t *atomic,
+ _odp_u128_t old_val,
+ _odp_u128_t new_val)
{
- return __lockfree_compare_exchange_16((__int128 *)atomic,
- (__int128 *)&old_val,
+ return __lockfree_compare_exchange_16((_odp_u128_t *)atomic,
+ (_odp_u128_t *)&old_val,
new_val,
0,
__ATOMIC_ACQ_REL,
@@ -249,7 +244,7 @@ static inline int lockfree_check_u128(void)
/** Atomic bit set operations with memory ordering */
#if defined(__SIZEOF_INT128__) && __SIZEOF_INT128__ == 16
-typedef __int128 bitset_t;
+typedef _odp_u128_t bitset_t;
#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT128__)
#elif __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \
@@ -292,11 +287,9 @@ static inline bitset_t bitset_mask(uint32_t bit)
if (bit < 64)
return 1ULL << bit;
else
- return (unsigned __int128)(1ULL << (bit - 64)) << 64;
+ return (_odp_u128_t)(1ULL << (bit - 64)) << 64;
}
-#pragma GCC diagnostic pop
-
#else
#error Unsupported size of bit sets (ATOM_BITSET_SIZE)
#endif
diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c
index d719c6cfa..4d46846ce 100644
--- a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c
+++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c
@@ -1,5 +1,6 @@
/* Copyright (c) 2014-2018, Linaro Limited
* Copyright (c) 2021, ARM Limited
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -37,9 +38,9 @@
/*
* ARM crypto library may read up to 15 bytes past the end of input
- * data and AAD.
+ * data and AAD and write up to 15 bytes past the end of output data.
*/
-#define OOB_READ_LEN 15
+#define OOB_WRITE_LEN 16 /* rounded up to 16 bytes for efficiency */
/*
* Data buffer size must be a multiple of 16, because the ARM crypto
@@ -80,9 +81,11 @@ static const odp_crypto_cipher_capability_t cipher_capa_aes_gcm[] = {
static const odp_crypto_auth_capability_t auth_capa_null[] = {
{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0} } };
+#define AES_GCM_TAG_LEN 16
+
#ifdef __ARM_FEATURE_AES
static const odp_crypto_auth_capability_t auth_capa_aes_gcm[] = {
-{.digest_len = 16, .key_len = 0, .aad_len = {.min = 8, .max = 12, .inc = 4} } };
+{.digest_len = AES_GCM_TAG_LEN, .key_len = 0, .aad_len = {.min = 8, .max = 12, .inc = 4} } };
#endif
/** Forward declaration of session structure */
@@ -92,9 +95,9 @@ typedef struct odp_crypto_generic_session_t odp_crypto_generic_session_t;
* Algorithm handler function prototype
*/
typedef
-odp_crypto_alg_err_t (*crypto_func_t)(odp_packet_t pkt,
- const odp_crypto_packet_op_param_t *param,
- odp_crypto_generic_session_t *session);
+odp_bool_t (*crypto_func_t)(odp_packet_t pkt,
+ const odp_crypto_packet_op_param_t *param,
+ odp_crypto_generic_session_t *session);
/**
* Per crypto session data structure
@@ -105,15 +108,12 @@ struct odp_crypto_generic_session_t {
/* Session creation parameters */
odp_crypto_session_param_t p;
- odp_bool_t do_cipher_first;
-
struct {
#if ODP_DEPRECATED_API
/* Copy of session IV data */
uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH];
#endif
uint8_t key_data[ARM_CRYPTO_MAX_CIPHER_KEY_LENGTH];
- crypto_func_t func;
} cipher;
struct {
@@ -121,11 +121,10 @@ struct odp_crypto_generic_session_t {
#if ODP_DEPRECATED_API
uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH];
#endif
- crypto_func_t func;
} auth;
+ crypto_func_t func;
unsigned int idx;
-
armv8_cipher_constants_t cc;
};
@@ -141,7 +140,6 @@ static odp_crypto_global_t *global;
typedef struct crypto_local_t {
uint8_t buffer[ARM_CRYPTO_MAX_DATA_LENGTH];
- uint8_t digest[ARM_CRYPTO_MAX_DIGEST_LENGTH];
} crypto_local_t;
static __thread crypto_local_t local;
@@ -176,18 +174,60 @@ void free_session(odp_crypto_generic_session_t *session)
odp_spinlock_unlock(&global->lock);
}
-static odp_crypto_alg_err_t
+static
+odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt)
+{
+ odp_packet_hdr_t *hdr = packet_hdr(pkt);
+
+ return &hdr->crypto_op_result;
+}
+
+static inline void set_crypto_op_result(odp_packet_t pkt,
+ odp_crypto_alg_err_t cipher_err,
+ odp_crypto_alg_err_t auth_err)
+{
+ odp_crypto_packet_result_t *op_result;
+
+ op_result = get_op_result_from_packet(pkt);
+ op_result->cipher_status.alg_err = cipher_err;
+ op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
+ op_result->auth_status.alg_err = auth_err;
+ op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
+ op_result->ok = (cipher_err == ODP_CRYPTO_ALG_ERR_NONE &&
+ auth_err == ODP_CRYPTO_ALG_ERR_NONE);
+}
+
+static inline void set_crypto_op_result_ok(odp_packet_t pkt)
+{
+ set_crypto_op_result(pkt,
+ ODP_CRYPTO_ALG_ERR_NONE,
+ ODP_CRYPTO_ALG_ERR_NONE);
+}
+
+static odp_bool_t
null_crypto_routine(odp_packet_t pkt ODP_UNUSED,
const odp_crypto_packet_op_param_t *param ODP_UNUSED,
odp_crypto_generic_session_t *session ODP_UNUSED)
{
- return ODP_CRYPTO_ALG_ERR_NONE;
+ set_crypto_op_result_ok(pkt);
+ return true;
+}
+
+static inline void copy_aad(uint8_t *dst, uint8_t *src, uint32_t len)
+{
+ ODP_ASSERT(len == 8 || len == 12);
+
+ /* Use constant length memcpy for better optimization result */
+ if (len == 8)
+ memcpy(dst, src, 8);
+ else
+ memcpy(dst, src, 12);
}
static
-odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
- const odp_crypto_packet_op_param_t *param,
- odp_crypto_generic_session_t *session)
+odp_bool_t aes_gcm_encrypt(odp_packet_t pkt,
+ const odp_crypto_packet_op_param_t *param,
+ odp_crypto_generic_session_t *session)
{
armv8_cipher_state_t cs = {
.counter = {
@@ -200,12 +240,14 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
uint64_t aad_bit_length = session->p.auth_aad_len * 8;
uint32_t in_pos = param->cipher_range.offset;
uint32_t in_len = param->cipher_range.length;
- int ret = 0;
+ odp_bool_t continuous_data;
+ uint16_t saved_tail[OOB_WRITE_LEN];
+ uint8_t tag[AES_GCM_TAG_LEN];
+ int rc;
/* Fail early if cipher_range is too large */
- if (in_len > ARM_CRYPTO_MAX_DATA_LENGTH) {
+ if (odp_unlikely(in_len > ARM_CRYPTO_MAX_DATA_LENGTH)) {
ODP_DBG("ARM Crypto: Packet size too large for requested operation\n");
- ret = -1;
goto err;
}
@@ -223,9 +265,9 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
cs.constants = &session->cc;
- if (armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs) != 0) {
+ rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs);
+ if (odp_unlikely(rc)) {
ODP_DBG("ARM Crypto: Failure while setting nonce\n");
- ret = -1;
goto err;
}
@@ -233,46 +275,60 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
* read it in 16 byte chunks. */
uint8_t aad[ARM_CRYPTO_MAX_AAD_LENGTH];
- memcpy(aad, param->aad_ptr, session->p.auth_aad_len);
+ copy_aad(aad, param->aad_ptr, session->p.auth_aad_len);
uint32_t seg_len = 0;
uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL);
if (odp_unlikely(odp_packet_is_segmented(pkt)) ||
- odp_unlikely(odp_packet_tailroom(pkt) < OOB_READ_LEN)) {
- /* Packet is segmented or it may not be safe to read beyond
- * the end of packet data. Copy the cipher range to a
+ odp_unlikely(odp_packet_tailroom(pkt) < OOB_WRITE_LEN)) {
+ /* Packet is segmented or it may not be safe to read and write
+ * beyond the end of packet data. Copy the cipher range to a
* contiguous buffer. */
odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer);
data = local.buffer;
+ continuous_data = false;
+ } else {
+ /* Save data that might get overwritten */
+ memcpy(saved_tail, data + in_len, OOB_WRITE_LEN);
+ continuous_data = true;
}
- if (armv8_enc_aes_gcm_from_state(&cs,
- aad, aad_bit_length,
- data, plaintext_bit_length,
- local.buffer,
- local.digest) != 0) {
+ rc = armv8_enc_aes_gcm_from_state(&cs,
+ aad, aad_bit_length,
+ data, plaintext_bit_length,
+ data,
+ tag);
+ if (odp_unlikely(rc)) {
ODP_DBG("ARM Crypto: AES GCM Encoding failed\n");
- ret = -1;
goto err;
}
- odp_packet_copy_from_mem(pkt, param->cipher_range.offset,
- param->cipher_range.length, local.buffer);
+ if (odp_likely(continuous_data)) {
+ memcpy(data + in_len, saved_tail, OOB_WRITE_LEN);
+ memcpy(data - in_pos + param->hash_result_offset,
+ tag, AES_GCM_TAG_LEN);
+ } else {
+ odp_packet_copy_from_mem(pkt, in_pos, in_len, data);
+ odp_packet_copy_from_mem(pkt, param->hash_result_offset,
+ AES_GCM_TAG_LEN, tag);
+ }
- odp_packet_copy_from_mem(pkt, param->hash_result_offset,
- session->p.auth_digest_len, local.digest);
+ set_crypto_op_result_ok(pkt);
+ return true;
err:
- return ret < 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE :
- ODP_CRYPTO_ALG_ERR_NONE;
+ set_crypto_op_result(pkt,
+ ODP_CRYPTO_ALG_ERR_DATA_SIZE,
+ ODP_CRYPTO_ALG_ERR_NONE);
+ return false;
}
static
-odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt,
- const odp_crypto_packet_op_param_t *param,
- odp_crypto_generic_session_t *session)
+odp_bool_t aes_gcm_decrypt(odp_packet_t pkt,
+ const odp_crypto_packet_op_param_t *param,
+ odp_crypto_generic_session_t *session)
{
armv8_cipher_state_t cs = {
.counter = {
@@ -280,18 +336,19 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt,
}
};
uint8_t *iv_ptr;
- uint8_t tag[16];
+ uint8_t tag[AES_GCM_TAG_LEN];
uint64_t iv_bit_length = session->p.cipher_iv_len * 8;
uint64_t plaintext_bit_length = param->cipher_range.length * 8;
uint64_t aad_bit_length = session->p.auth_aad_len * 8;
uint32_t in_pos = param->cipher_range.offset;
uint32_t in_len = param->cipher_range.length;
- int ret = 0;
+ odp_bool_t continuous_data;
+ uint16_t saved_tail[OOB_WRITE_LEN];
+ int rc;
/* Fail early if cipher_range is too large */
- if (in_len > ARM_CRYPTO_MAX_DATA_LENGTH) {
+ if (odp_unlikely(in_len > ARM_CRYPTO_MAX_DATA_LENGTH)) {
ODP_DBG("ARM Crypto: Packet size too large for requested operation\n");
- ret = -1;
goto err;
}
@@ -309,51 +366,63 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt,
cs.constants = &session->cc;
- if (armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs) != 0) {
+ rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs);
+ if (odp_unlikely(rc)) {
ODP_DBG("ARM Crypto: Failure while setting nonce\n");
- ret = -1;
goto err;
}
- /* Copy current hash bytes to a buffer and clear it */
- odp_packet_copy_to_mem(pkt, param->hash_result_offset,
- session->p.auth_digest_len, tag);
-
/* Copy AAD in a stack to make sure that the ARM crypto library can
* read it in 16 byte chunks. */
uint8_t aad[ARM_CRYPTO_MAX_AAD_LENGTH];
- memcpy(aad, param->aad_ptr, session->p.auth_aad_len);
+ copy_aad(aad, param->aad_ptr, session->p.auth_aad_len);
uint32_t seg_len = 0;
uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL);
if (odp_unlikely(odp_packet_is_segmented(pkt)) ||
- odp_unlikely(odp_packet_tailroom(pkt) < OOB_READ_LEN)) {
- /* Packet is segmented or it may not be safe to read beyond
- * the end of packet data. Copy the cipher range to a
+ odp_unlikely(odp_packet_tailroom(pkt) < OOB_WRITE_LEN)) {
+ /* Packet is segmented or it may not be safe to read and write
+ * beyond the end of packet data. Copy the cipher range to a
* contiguous buffer. */
odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer);
-
data = local.buffer;
+ /* Copy tag from the packet to a buffer */
+ odp_packet_copy_to_mem(pkt, param->hash_result_offset,
+ AES_GCM_TAG_LEN, tag);
+ continuous_data = false;
+ } else {
+ /* Save data that might get overwritten */
+ memcpy(saved_tail, data + in_len, OOB_WRITE_LEN);
+ /* Copy tag from the packet to a buffer */
+ memcpy(tag, data - in_pos + param->hash_result_offset, AES_GCM_TAG_LEN);
+ continuous_data = true;
}
- if (armv8_dec_aes_gcm_from_state(&cs,
- aad, aad_bit_length,
- data, plaintext_bit_length,
- tag,
- local.buffer) != 0) {
+ rc = armv8_dec_aes_gcm_from_state(&cs,
+ aad, aad_bit_length,
+ data, plaintext_bit_length,
+ tag,
+ data);
+ if (odp_unlikely(rc)) {
ODP_DBG("ARM Crypto: AES GCM Decoding failed\n");
- ret = -1;
goto err;
}
- odp_packet_copy_from_mem(pkt, param->cipher_range.offset,
- param->cipher_range.length, local.buffer);
+ if (odp_likely(continuous_data))
+ memcpy(data + in_len, saved_tail, OOB_WRITE_LEN);
+ else
+ odp_packet_copy_from_mem(pkt, in_pos, in_len, data);
+
+ set_crypto_op_result_ok(pkt);
+ return true;
err:
- return ret < 0 ? ODP_CRYPTO_ALG_ERR_ICV_CHECK :
- ODP_CRYPTO_ALG_ERR_NONE;
+ set_crypto_op_result(pkt,
+ ODP_CRYPTO_ALG_ERR_NONE,
+ ODP_CRYPTO_ALG_ERR_ICV_CHECK);
+ return false;
}
static int process_aes_gcm_param(odp_crypto_generic_session_t *session)
@@ -376,9 +445,9 @@ static int process_aes_gcm_param(odp_crypto_generic_session_t *session)
/* Set function */
if (ODP_CRYPTO_OP_ENCODE == session->p.op)
- session->cipher.func = aes_gcm_encrypt;
+ session->func = aes_gcm_encrypt;
else
- session->cipher.func = aes_gcm_decrypt;
+ session->func = aes_gcm_decrypt;
return 0;
}
@@ -520,16 +589,10 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
session->p.auth_iv.length);
#endif
- /* Derive order */
- if (ODP_CRYPTO_OP_ENCODE == param->op)
- session->do_cipher_first = param->auth_cipher_text;
- else
- session->do_cipher_first = !param->auth_cipher_text;
-
/* Process based on cipher */
switch (param->cipher_alg) {
case ODP_CIPHER_ALG_NULL:
- session->cipher.func = null_crypto_routine;
+ session->func = null_crypto_routine;
rc = 0;
break;
case ODP_CIPHER_ALG_AES_GCM:
@@ -584,14 +647,15 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
/* Process based on auth */
switch (param->auth_alg) {
case ODP_AUTH_ALG_NULL:
- session->auth.func = null_crypto_routine;
- rc = 0;
+ if (param->cipher_alg == ODP_CIPHER_ALG_NULL)
+ rc = 0;
+ else
+ rc = -1;
break;
case ODP_AUTH_ALG_AES_GCM:
/* AES-GCM requires to do both auth and
* cipher at the same time */
if (param->cipher_alg == ODP_CIPHER_ALG_AES_GCM) {
- session->auth.func = null_crypto_routine;
rc = 0;
} else {
rc = -1;
@@ -630,9 +694,7 @@ int odp_crypto_session_destroy(odp_crypto_session_t session)
return 0;
}
-/*
- * Shim function around packet operation, can be used by other implementations.
- */
+#if ODP_DEPRECATED_API
int
odp_crypto_operation(odp_crypto_op_param_t *param,
odp_bool_t *posted,
@@ -687,6 +749,7 @@ odp_crypto_operation(odp_crypto_op_param_t *param,
return 0;
}
+#endif
int _odp_crypto_init_global(void)
{
@@ -767,6 +830,7 @@ int _odp_crypto_term_local(void)
return 0;
}
+#if ODP_DEPRECATED_API
odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev)
{
/* This check not mandated by the API specification */
@@ -803,6 +867,7 @@ uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl)
{
return _odp_pri(hdl);
}
+#endif /* ODP_DEPRECATED_API */
void odp_crypto_session_param_init(odp_crypto_session_param_t *param)
{
@@ -828,14 +893,6 @@ odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt)
return odp_packet_to_event(pkt);
}
-static
-odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt)
-{
- odp_packet_hdr_t *hdr = packet_hdr(pkt);
-
- return &hdr->crypto_op_result;
-}
-
int odp_crypto_result(odp_crypto_packet_result_t *result,
odp_packet_t packet)
{
@@ -856,18 +913,16 @@ int crypto_int(odp_packet_t pkt_in,
odp_packet_t *pkt_out,
const odp_crypto_packet_op_param_t *param)
{
- odp_crypto_alg_err_t rc_cipher = ODP_CRYPTO_ALG_ERR_NONE;
- odp_crypto_alg_err_t rc_auth = ODP_CRYPTO_ALG_ERR_NONE;
odp_crypto_generic_session_t *session;
odp_bool_t allocated = false;
odp_packet_t out_pkt = *pkt_out;
- odp_crypto_packet_result_t *op_result;
odp_packet_hdr_t *pkt_hdr;
+ odp_bool_t ok;
session = (odp_crypto_generic_session_t *)(intptr_t)param->session;
/* Resolve output buffer */
- if (ODP_PACKET_INVALID == out_pkt &&
+ if (odp_unlikely(ODP_PACKET_INVALID == out_pkt) &&
ODP_POOL_INVALID != session->p.output_pool) {
out_pkt = odp_packet_alloc(session->p.output_pool,
odp_packet_len(pkt_in));
@@ -879,7 +934,7 @@ int crypto_int(odp_packet_t pkt_in,
return -1;
}
- if (pkt_in != out_pkt) {
+ if (odp_unlikely(pkt_in != out_pkt)) {
int ret;
int md_copy;
@@ -903,28 +958,12 @@ int crypto_int(odp_packet_t pkt_in,
pkt_in = ODP_PACKET_INVALID;
}
- /* Invoke the functions */
- if (session->do_cipher_first) {
- rc_cipher = session->cipher.func(out_pkt, param, session);
- rc_auth = session->auth.func(out_pkt, param, session);
- } else {
- rc_auth = session->auth.func(out_pkt, param, session);
- rc_cipher = session->cipher.func(out_pkt, param, session);
- }
+ /* Invoke the crypto function */
+ ok = session->func(out_pkt, param, session);
- /* Fill in result */
packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO);
- op_result = get_op_result_from_packet(out_pkt);
- op_result->cipher_status.alg_err = rc_cipher;
- op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
- op_result->auth_status.alg_err = rc_auth;
- op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
- op_result->ok =
- (rc_cipher == ODP_CRYPTO_ALG_ERR_NONE) &&
- (rc_auth == ODP_CRYPTO_ALG_ERR_NONE);
-
pkt_hdr = packet_hdr(out_pkt);
- pkt_hdr->p.flags.crypto_err = !op_result->ok;
+ pkt_hdr->p.flags.crypto_err = !ok;
/* Synchronous, simply return results */
*pkt_out = out_pkt;
diff --git a/platform/linux-generic/arch/aarch64/odp_llsc.h b/platform/linux-generic/arch/aarch64/odp_llsc.h
index 2561451ca..1c2610511 100644
--- a/platform/linux-generic/arch/aarch64/odp_llsc.h
+++ b/platform/linux-generic/arch/aarch64/odp_llsc.h
@@ -13,6 +13,8 @@
#error This file should not be included directly, please include odp_cpu.h
#endif
+#include <odp_types_internal.h>
+
static inline uint16_t ll8(uint8_t *var, int mm)
{
uint16_t old;
@@ -115,11 +117,11 @@ static inline uint32_t sc(uint64_t *var, uint64_t neu, int mm)
#define sc64(a, b, c) sc((a), (b), (c))
union i128 {
- __extension__ __int128 i128;
+ _odp_u128_t i128;
int64_t i64[2];
};
-__extension__ static inline __int128 lld(__int128 *var, int mm)
+static inline _odp_u128_t lld(_odp_u128_t *var, int mm)
{
union i128 old;
@@ -139,7 +141,7 @@ __extension__ static inline __int128 lld(__int128 *var, int mm)
}
/* Return 0 on success, 1 on failure */
-__extension__ static inline uint32_t scd(__int128 *var, __int128 neu, int mm)
+static inline uint32_t scd(_odp_u128_t *var, _odp_u128_t neu, int mm)
{
uint32_t ret;
diff --git a/platform/linux-generic/arch/arm/odp_atomic.h b/platform/linux-generic/arch/arm/odp_atomic.h
index bcc89ff73..e400f52d4 100644
--- a/platform/linux-generic/arch/arm/odp_atomic.h
+++ b/platform/linux-generic/arch/arm/odp_atomic.h
@@ -12,6 +12,7 @@
#error This file should not be included directly, please include odp_cpu.h
#endif
+#include <odp_types_internal.h>
#include <limits.h>
#ifdef CONFIG_DMBSTR
@@ -70,7 +71,7 @@ static inline bitset_t bitset_mask(uint32_t bit)
if (bit < 64)
return 1ULL << bit;
else
- return (unsigned __int128)(1ULL << (bit - 64)) << 64;
+ return (_odp_u128_t)(1ULL << (bit - 64)) << 64;
}
#else
diff --git a/platform/linux-generic/arch/default/odp_atomic.h b/platform/linux-generic/arch/default/odp_atomic.h
index 7650d2b52..4cfc6b4bd 100644
--- a/platform/linux-generic/arch/default/odp_atomic.h
+++ b/platform/linux-generic/arch/default/odp_atomic.h
@@ -7,18 +7,18 @@
#ifndef ODP_DEFAULT_ATOMIC_H_
#define ODP_DEFAULT_ATOMIC_H_
-#ifdef __SIZEOF_INT128__
+#include <odp_types_internal.h>
-__extension__ typedef unsigned __int128 _u128_t;
+#ifdef __SIZEOF_INT128__
-static inline _u128_t lockfree_load_u128(_u128_t *atomic)
+static inline _odp_u128_t lockfree_load_u128(_odp_u128_t *atomic)
{
return __atomic_load_n(atomic, __ATOMIC_RELAXED);
}
-static inline int lockfree_cas_acq_rel_u128(_u128_t *atomic,
- _u128_t old_val,
- _u128_t new_val)
+static inline int lockfree_cas_acq_rel_u128(_odp_u128_t *atomic,
+ _odp_u128_t old_val,
+ _odp_u128_t new_val)
{
return __atomic_compare_exchange_n(atomic, &old_val, new_val,
0 /* strong */,
@@ -76,7 +76,7 @@ static inline bitset_t bitset_mask(uint32_t bit)
if (bit < 64)
return 1ULL << bit;
else
- return (unsigned __int128)(1ULL << (bit - 64)) << 64;
+ return (_odp_u128_t)(1ULL << (bit - 64)) << 64;
}
#else
diff --git a/platform/linux-generic/include-abi/odp/api/abi/event.h b/platform/linux-generic/include-abi/odp/api/abi/event.h
index 6530ac2e4..e059f318c 100644
--- a/platform/linux-generic/include-abi/odp/api/abi/event.h
+++ b/platform/linux-generic/include-abi/odp/api/abi/event.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2015-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -18,42 +19,9 @@
extern "C" {
#endif
-#include <odp/api/plat/strong_types.h>
-
-/** @ingroup odp_event
- * @{
- */
-
-typedef ODP_HANDLE_T(odp_event_t);
-
-#define ODP_EVENT_INVALID _odp_cast_scalar(odp_event_t, 0)
-
-typedef enum odp_event_type_t {
- ODP_EVENT_BUFFER = 1,
- ODP_EVENT_PACKET = 2,
- ODP_EVENT_TIMEOUT = 3,
- ODP_EVENT_CRYPTO_COMPL = 4,
- ODP_EVENT_IPSEC_STATUS = 5,
- ODP_EVENT_PACKET_VECTOR = 6,
- ODP_EVENT_PACKET_TX_COMPL = 7,
- ODP_EVENT_DMA_COMPL = 8,
-} odp_event_type_t;
-
-typedef enum odp_event_subtype_t {
- ODP_EVENT_NO_SUBTYPE = 0,
- ODP_EVENT_PACKET_BASIC = 1,
- ODP_EVENT_PACKET_CRYPTO = 2,
- ODP_EVENT_PACKET_IPSEC = 3,
- ODP_EVENT_PACKET_COMP = 4
-} odp_event_subtype_t;
-
/* Inlined functions for non-ABI compat mode */
#include <odp/api/plat/event_inlines.h>
-/**
- * @}
- */
-
#ifdef __cplusplus
}
#endif
diff --git a/platform/linux-generic/include-abi/odp/api/abi/event_types.h b/platform/linux-generic/include-abi/odp/api/abi/event_types.h
new file mode 100644
index 000000000..c39322330
--- /dev/null
+++ b/platform/linux-generic/include-abi/odp/api/abi/event_types.h
@@ -0,0 +1,61 @@
+/* Copyright (c) 2015-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP event type definitions
+ */
+
+#ifndef ODP_API_ABI_EVENT_TYPES_H_
+#define ODP_API_ABI_EVENT_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/api/plat/strong_types.h>
+#include <odp/api/deprecated.h>
+
+/** @ingroup odp_event
+ * @{
+ */
+
+typedef ODP_HANDLE_T(odp_event_t);
+
+#define ODP_EVENT_INVALID _odp_cast_scalar(odp_event_t, 0)
+
+typedef enum odp_event_type_t {
+ ODP_EVENT_BUFFER = 1,
+ ODP_EVENT_PACKET = 2,
+ ODP_EVENT_TIMEOUT = 3,
+#if ODP_DEPRECATED_API
+ ODP_EVENT_CRYPTO_COMPL = 4,
+#endif
+ ODP_EVENT_IPSEC_STATUS = 5,
+ ODP_EVENT_PACKET_VECTOR = 6,
+ ODP_EVENT_PACKET_TX_COMPL = 7,
+ ODP_EVENT_DMA_COMPL = 8,
+} odp_event_type_t;
+
+typedef enum odp_event_subtype_t {
+ ODP_EVENT_NO_SUBTYPE = 0,
+ ODP_EVENT_PACKET_BASIC = 1,
+ ODP_EVENT_PACKET_CRYPTO = 2,
+ ODP_EVENT_PACKET_IPSEC = 3,
+ ODP_EVENT_PACKET_COMP = 4
+} odp_event_subtype_t;
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include-abi/odp/api/abi/timer.h b/platform/linux-generic/include-abi/odp/api/abi/timer.h
new file mode 100644
index 000000000..0a3b3a9cc
--- /dev/null
+++ b/platform/linux-generic/include-abi/odp/api/abi/timer.h
@@ -0,0 +1,8 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/* Inlined functions for non-ABI compat mode */
+#include <odp/api/plat/timer_inlines.h>
diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h
index 632ba0e69..9e0ae3f50 100644
--- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h
@@ -8,7 +8,7 @@
#define ODP_PLAT_BUFFER_INLINES_H_
#include <odp/api/abi/buffer.h>
-#include <odp/api/abi/event.h>
+#include <odp/api/abi/event_types.h>
#include <odp/api/plat/buffer_inline_types.h>
diff --git a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h
index 6eb34a39b..04ad9e968 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2017-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -30,11 +31,32 @@ static inline uint64_t _odp_packet_input_flags(odp_packet_t pkt)
/* Inline functions by default */
#define _ODP_INLINE static inline
#define odp_packet_has_l2 __odp_packet_has_l2
+ #define odp_packet_has_l3 __odp_packet_has_l3
+ #define odp_packet_has_l4 __odp_packet_has_l4
#define odp_packet_has_eth __odp_packet_has_eth
#define odp_packet_has_jumbo __odp_packet_has_jumbo
#define odp_packet_has_flow_hash __odp_packet_has_flow_hash
#define odp_packet_has_ts __odp_packet_has_ts
#define odp_packet_has_ipsec __odp_packet_has_ipsec
+ #define odp_packet_has_eth_bcast __odp_packet_has_eth_bcast
+ #define odp_packet_has_eth_mcast __odp_packet_has_eth_mcast
+ #define odp_packet_has_vlan __odp_packet_has_vlan
+ #define odp_packet_has_vlan_qinq __odp_packet_has_vlan_qinq
+ #define odp_packet_has_arp __odp_packet_has_arp
+ #define odp_packet_has_ipv4 __odp_packet_has_ipv4
+ #define odp_packet_has_ipv6 __odp_packet_has_ipv6
+ #define odp_packet_has_ip_bcast __odp_packet_has_ip_bcast
+ #define odp_packet_has_ip_mcast __odp_packet_has_ip_mcast
+ #define odp_packet_has_ipfrag __odp_packet_has_ipfrag
+ #define odp_packet_has_ipopt __odp_packet_has_ipopt
+ #define odp_packet_has_udp __odp_packet_has_udp
+ #define odp_packet_has_tcp __odp_packet_has_tcp
+ #define odp_packet_has_sctp __odp_packet_has_sctp
+ #define odp_packet_has_icmp __odp_packet_has_icmp
+ #define odp_packet_has_error __odp_packet_has_error
+ #define odp_packet_has_l2_error __odp_packet_has_l2_error
+ #define odp_packet_has_l3_error __odp_packet_has_l3_error
+ #define odp_packet_has_l4_error __odp_packet_has_l4_error
#else
#undef _ODP_INLINE
#define _ODP_INLINE
@@ -48,6 +70,22 @@ _ODP_INLINE int odp_packet_has_l2(odp_packet_t pkt)
return flags.l2;
}
+_ODP_INLINE int odp_packet_has_l3(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.l3;
+}
+
+_ODP_INLINE int odp_packet_has_l4(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.l4;
+}
+
_ODP_INLINE int odp_packet_has_eth(odp_packet_t pkt)
{
_odp_packet_input_flags_t flags;
@@ -88,6 +126,163 @@ _ODP_INLINE int odp_packet_has_ipsec(odp_packet_t pkt)
return flags.ipsec;
}
+_ODP_INLINE int odp_packet_has_eth_bcast(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.eth_bcast;
+}
+
+_ODP_INLINE int odp_packet_has_eth_mcast(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.eth_mcast;
+}
+
+_ODP_INLINE int odp_packet_has_vlan(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.vlan;
+}
+
+_ODP_INLINE int odp_packet_has_vlan_qinq(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.vlan_qinq;
+}
+
+_ODP_INLINE int odp_packet_has_arp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.arp;
+}
+
+_ODP_INLINE int odp_packet_has_ipv4(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ipv4;
+}
+
+_ODP_INLINE int odp_packet_has_ipv6(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ipv6;
+}
+
+_ODP_INLINE int odp_packet_has_ip_bcast(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ip_bcast;
+}
+
+_ODP_INLINE int odp_packet_has_ip_mcast(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ip_mcast;
+}
+
+_ODP_INLINE int odp_packet_has_ipfrag(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ipfrag;
+}
+
+_ODP_INLINE int odp_packet_has_ipopt(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ipopt;
+}
+
+_ODP_INLINE int odp_packet_has_udp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.udp;
+}
+
+_ODP_INLINE int odp_packet_has_tcp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.tcp;
+}
+
+_ODP_INLINE int odp_packet_has_sctp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.sctp;
+}
+
+_ODP_INLINE int odp_packet_has_icmp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.icmp;
+}
+
+_ODP_INLINE int odp_packet_has_error(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+ return flags.all.error != 0;
+}
+
+_ODP_INLINE int odp_packet_has_l2_error(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+
+ /* L2 parsing is always done by default and hence
+ no additional check is required. */
+ return flags.snap_len_err;
+}
+
+_ODP_INLINE int odp_packet_has_l3_error(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+
+ return flags.ip_err;
+}
+
+_ODP_INLINE int odp_packet_has_l4_error(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+
+ return flags.tcp_err | flags.udp_err;
+}
+
/** @endcond */
#endif
diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h
index 66451ce49..126404f2d 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h
@@ -115,7 +115,7 @@ typedef union {
uint32_t all_flags;
struct {
- uint32_t reserved1: 7;
+ uint32_t reserved1: 6;
/*
* Init flags
@@ -133,6 +133,7 @@ typedef union {
uint32_t l4_chksum: 1; /* L4 chksum override */
uint32_t ts_set: 1; /* Set Tx timestamp */
uint32_t tx_compl: 1; /* Tx completion event requested */
+ uint32_t tx_aging: 1; /* Packet aging at Tx requested */
uint32_t shaper_len_adj: 8; /* Adjustment for traffic mgr */
/*
@@ -150,8 +151,8 @@ typedef union {
/* Flag groups */
struct {
- uint32_t reserved2: 7;
- uint32_t other: 17; /* All other flags */
+ uint32_t reserved2: 6;
+ uint32_t other: 18; /* All other flags */
uint32_t error: 8; /* All error flags */
} all;
diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h
index fb63f207b..b72749ea8 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h
@@ -20,7 +20,7 @@
#include <odp/api/time.h>
#include <odp/api/abi/buffer.h>
-#include <odp/api/abi/event.h>
+#include <odp/api/abi/event_types.h>
#include <odp/api/abi/packet.h>
#include <odp/api/abi/packet_io.h>
@@ -73,6 +73,9 @@
#define odp_packet_subtype __odp_packet_subtype
#define odp_packet_tx_compl_from_event __odp_packet_tx_compl_from_event
#define odp_packet_tx_compl_to_event __odp_packet_tx_compl_to_event
+ #define odp_packet_color __odp_packet_color
+ #define odp_packet_drop_eligible __odp_packet_drop_eligible
+ #define odp_packet_shaper_len_adjust __odp_packet_shaper_len_adjust
#else
#undef _ODP_INLINE
#define _ODP_INLINE
@@ -372,6 +375,33 @@ _ODP_INLINE odp_event_t odp_packet_tx_compl_to_event(odp_packet_tx_compl_t tx_co
return (odp_event_t)(uintptr_t)tx_compl;
}
+_ODP_INLINE odp_packet_color_t odp_packet_color(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t input_flags;
+
+ input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags);
+
+ return (odp_packet_color_t)input_flags.color;
+}
+
+_ODP_INLINE odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t input_flags;
+
+ input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags);
+
+ return !input_flags.nodrop;
+}
+
+_ODP_INLINE int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+
+ return flags.shaper_len_adj;
+}
+
/** @endcond */
#endif
diff --git a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h
index 3586aa6d5..54a009a36 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h
@@ -14,7 +14,7 @@
#ifndef _ODP_PLAT_PACKET_VECTOR_INLINES_H_
#define _ODP_PLAT_PACKET_VECTOR_INLINES_H_
-#include <odp/api/abi/event.h>
+#include <odp/api/abi/event_types.h>
#include <odp/api/abi/packet_types.h>
#include <odp/api/abi/pool_types.h>
diff --git a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h
index 5ce767a16..4eade3dea 100644
--- a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h
+++ b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h
@@ -12,6 +12,7 @@ extern "C" {
#endif
#include <stdint.h>
+#include <odp/api/event_types.h>
#include <odp/api/queue_types.h>
/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
diff --git a/platform/linux-generic/include/odp/api/plat/time_inlines.h b/platform/linux-generic/include/odp/api/plat/time_inlines.h
index bb2913532..2ffb94c66 100644
--- a/platform/linux-generic/include/odp/api/plat/time_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/time_inlines.h
@@ -1,5 +1,5 @@
/* Copyright (c) 2018, Linaro Limited
- * Copyright (c) 2020-2021, Nokia
+ * Copyright (c) 2020-2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -99,6 +99,7 @@ static inline uint64_t _odp_time_convert_to_ns(odp_time_t time)
#define odp_time_cmp __odp_time_cmp
#define odp_time_diff __odp_time_diff
+ #define odp_time_diff_ns __odp_time_diff_ns
#define odp_time_sum __odp_time_sum
#else
@@ -170,6 +171,15 @@ _ODP_INLINE odp_time_t odp_time_diff(odp_time_t t2, odp_time_t t1)
return time;
}
+_ODP_INLINE uint64_t odp_time_diff_ns(odp_time_t t2, odp_time_t t1)
+{
+ odp_time_t time;
+
+ time.u64 = t2.u64 - t1.u64;
+
+ return odp_time_to_ns(time);
+}
+
_ODP_INLINE odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2)
{
odp_time_t time;
diff --git a/platform/linux-generic/include/odp/api/plat/timer_inline_types.h b/platform/linux-generic/include/odp/api/plat/timer_inline_types.h
new file mode 100644
index 000000000..e3397c4df
--- /dev/null
+++ b/platform/linux-generic/include/odp/api/plat/timer_inline_types.h
@@ -0,0 +1,37 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_PLAT_TIMER_INLINE_TYPES_H_
+#define ODP_PLAT_TIMER_INLINE_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
+
+/* Timeout header field accessor */
+#define _odp_timeout_hdr_field(hdr, cast, field) \
+ (*(cast *)(uintptr_t)((uint8_t *)hdr + \
+ _odp_timeout_inline_offset.field))
+
+/* Timeout header field offsets for inline functions */
+typedef struct _odp_timeout_inline_offset_t {
+ uint16_t expiration;
+ uint16_t timer;
+ uint16_t user_ptr;
+
+} _odp_timeout_inline_offset_t;
+
+/** @endcond */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include/odp/api/plat/timer_inlines.h b/platform/linux-generic/include/odp/api/plat/timer_inlines.h
new file mode 100644
index 000000000..406aefdf6
--- /dev/null
+++ b/platform/linux-generic/include/odp/api/plat/timer_inlines.h
@@ -0,0 +1,78 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_PLAT_TIMER_INLINES_H_
+#define ODP_PLAT_TIMER_INLINES_H_
+
+#include <odp/api/event.h>
+#include <odp/api/timer_types.h>
+
+#include <odp/api/plat/timer_inline_types.h>
+
+#include <stdint.h>
+
+/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
+
+extern const _odp_timeout_inline_offset_t _odp_timeout_inline_offset;
+
+#ifndef _ODP_NO_INLINE
+ /* Inline functions by default */
+ #define _ODP_INLINE static inline
+ #define odp_timeout_timer __odp_timeout_timer
+ #define odp_timeout_tick __odp_timeout_tick
+ #define odp_timeout_user_ptr __odp_timeout_user_ptr
+ #define odp_timer_tick_to_ns __odp_timer_tick_to_ns
+ #define odp_timer_ns_to_tick __odp_timer_ns_to_tick
+ #define odp_timeout_from_event __odp_timeout_from_event
+ #define odp_timeout_to_event __odp_timeout_to_event
+#else
+ #define _ODP_INLINE
+#endif
+
+_ODP_INLINE odp_timer_t odp_timeout_timer(odp_timeout_t tmo)
+{
+ return _odp_timeout_hdr_field(tmo, odp_timer_t, timer);
+}
+
+_ODP_INLINE uint64_t odp_timeout_tick(odp_timeout_t tmo)
+{
+ return _odp_timeout_hdr_field(tmo, uint64_t, expiration);
+}
+
+_ODP_INLINE void *odp_timeout_user_ptr(odp_timeout_t tmo)
+{
+ return _odp_timeout_hdr_field(tmo, void *, user_ptr);
+}
+
+_ODP_INLINE uint64_t odp_timer_tick_to_ns(odp_timer_pool_t tp, uint64_t ticks)
+{
+ (void)tp;
+
+ /* Timer ticks in API are nsec */
+ return ticks;
+}
+
+_ODP_INLINE uint64_t odp_timer_ns_to_tick(odp_timer_pool_t tp, uint64_t ns)
+{
+ (void)tp;
+
+ /* Timer ticks in API are nsec */
+ return ns;
+}
+
+_ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev)
+{
+ return (odp_timeout_t)ev;
+}
+
+_ODP_INLINE odp_event_t odp_timeout_to_event(odp_timeout_t tmo)
+{
+ return (odp_event_t)tmo;
+}
+
+/** @endcond */
+
+#endif
diff --git a/platform/linux-generic/include/odp_atomic_internal.h b/platform/linux-generic/include/odp_atomic_internal.h
index d98e0e6be..da79b3723 100644
--- a/platform/linux-generic/include/odp_atomic_internal.h
+++ b/platform/linux-generic/include/odp_atomic_internal.h
@@ -20,6 +20,7 @@
#include <odp/api/align.h>
#include <odp/api/hints.h>
#include <odp/api/atomic.h>
+#include <odp_types_internal.h>
#include <stdbool.h>
#ifdef __cplusplus
@@ -137,12 +138,10 @@ static inline void _odp_atomic_flag_clear(_odp_atomic_flag_t *flag)
#endif
#ifdef ODP_ATOMIC_U128
-/** An unsigned 128-bit (16-byte) scalar type */
-__extension__ typedef __int128 _uint128_t;
/** Atomic 128-bit type */
typedef struct ODP_ALIGNED(16) {
- _uint128_t v; /**< Actual storage for the atomic variable */
+ _odp_u128_t v; /**< Actual storage for the atomic variable */
} _odp_atomic_u128_t;
/**
@@ -154,8 +153,8 @@ typedef struct ODP_ALIGNED(16) {
* @param mmodel Memory model associated with the exchange operation
*/
static inline void _odp_atomic_u128_xchg_mm(_odp_atomic_u128_t *ptr,
- _uint128_t *val,
- _uint128_t *old,
+ _odp_u128_t *val,
+ _odp_u128_t *old,
_odp_memmodel_t mm)
{
__atomic_exchange(&ptr->v, val, old, mm);
@@ -177,8 +176,8 @@ static inline void _odp_atomic_u128_xchg_mm(_odp_atomic_u128_t *ptr,
* @retval 0 exchange failed and '*exp' updated with current value
*/
static inline int _odp_atomic_u128_cmp_xchg_mm(_odp_atomic_u128_t *ptr,
- _uint128_t *exp,
- _uint128_t *val,
+ _odp_u128_t *exp,
+ _odp_u128_t *val,
_odp_memmodel_t succ,
_odp_memmodel_t fail)
{
diff --git a/platform/linux-generic/include/odp_classification_datamodel.h b/platform/linux-generic/include/odp_classification_datamodel.h
index cc0e7f081..6b50fef68 100644
--- a/platform/linux-generic/include/odp_classification_datamodel.h
+++ b/platform/linux-generic/include/odp_classification_datamodel.h
@@ -129,6 +129,7 @@ typedef struct pmr_term_value {
Class Of Service
*/
struct cos_s {
+ odp_cos_action_t action; /* Action */
odp_queue_t queue; /* Associated Queue */
odp_pool_t pool; /* Associated Buffer pool */
odp_pktin_vector_config_t vector; /* Packet vector config */
@@ -145,10 +146,11 @@ struct cos_s {
odp_queue_param_t queue_param;
char name[ODP_COS_NAME_LEN]; /* name */
uint8_t index;
+ odp_bool_t stats_enable;
struct {
odp_atomic_u64_t discards;
odp_atomic_u64_t packets;
- } stats[CLS_COS_QUEUE_MAX];
+ } stats, queue_stats[CLS_COS_QUEUE_MAX];
};
typedef union cos_u {
diff --git a/platform/linux-generic/include/odp_classification_internal.h b/platform/linux-generic/include/odp_classification_internal.h
index bc0a12f8f..c20d24c58 100644
--- a/platform/linux-generic/include/odp_classification_internal.h
+++ b/platform/linux-generic/include/odp_classification_internal.h
@@ -67,9 +67,9 @@ static inline void _odp_cos_queue_stats_add(cos_t *cos, odp_queue_t queue,
}
if (packets)
- odp_atomic_add_u64(&cos->s.stats[queue_idx].packets, packets);
+ odp_atomic_add_u64(&cos->s.queue_stats[queue_idx].packets, packets);
if (discards)
- odp_atomic_add_u64(&cos->s.stats[queue_idx].discards, discards);
+ odp_atomic_add_u64(&cos->s.queue_stats[queue_idx].discards, discards);
}
/** Classification Internal function **/
@@ -84,8 +84,7 @@ This function calls Classifier module internal functions for a given packet and
selects destination queue and packet pool based on selected PMR and CoS.
**/
int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base,
- uint16_t pkt_len, uint32_t seg_len, odp_pool_t *pool,
- odp_packet_hdr_t *pkt_hdr, odp_bool_t parse);
+ odp_pool_t *pool, odp_packet_hdr_t *pkt_hdr);
/**
Packet IO classifier init
diff --git a/platform/linux-generic/include/odp_llqueue.h b/platform/linux-generic/include/odp_llqueue.h
index e9cf9945e..68325624a 100644
--- a/platform/linux-generic/include/odp_llqueue.h
+++ b/platform/linux-generic/include/odp_llqueue.h
@@ -15,6 +15,7 @@
#include <odp_config_internal.h>
#include <odp_debug_internal.h>
+#include <odp_types_internal.h>
#include <odp_cpu.h>
#include <stdint.h>
@@ -49,7 +50,7 @@ static odp_bool_t llq_on_queue(struct llnode *node);
typedef uint64_t dintptr_t;
#endif
#if __SIZEOF_PTRDIFF_T__ == 8
-__extension__ typedef __int128 dintptr_t;
+typedef _odp_u128_t dintptr_t;
#endif
struct llnode {
diff --git a/platform/linux-generic/include/odp_macros_internal.h b/platform/linux-generic/include/odp_macros_internal.h
index 997e0fd5b..b8be7f938 100644
--- a/platform/linux-generic/include/odp_macros_internal.h
+++ b/platform/linux-generic/include/odp_macros_internal.h
@@ -35,6 +35,8 @@ extern "C" {
tmp_a > tmp_b ? tmp_a : tmp_b; \
})
+#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c)))
+
#define odp_container_of(pointer, type, member) \
((type *)(void *)(((char *)pointer) - offsetof(type, member)))
diff --git a/platform/linux-generic/include/odp_packet_dpdk.h b/platform/linux-generic/include/odp_packet_dpdk.h
index 3add83f19..b326000e6 100644
--- a/platform/linux-generic/include/odp_packet_dpdk.h
+++ b/platform/linux-generic/include/odp_packet_dpdk.h
@@ -46,18 +46,4 @@ int _odp_dpdk_packet_parse_common(packet_parser_t *pkt_hdr,
uint32_t supported_ptypes,
odp_pktin_config_opt_t pktin_cfg);
-static inline int _odp_dpdk_packet_parse_layer(odp_packet_hdr_t *pkt_hdr,
- struct rte_mbuf *mbuf,
- odp_proto_layer_t layer,
- uint32_t supported_ptypes,
- odp_pktin_config_opt_t pktin_cfg)
-{
- uint32_t seg_len = pkt_hdr->seg_len;
- void *base = pkt_hdr->seg_data;
-
- return _odp_dpdk_packet_parse_common(&pkt_hdr->p, base,
- pkt_hdr->frame_len, seg_len, mbuf,
- layer, supported_ptypes,
- pktin_cfg);
-}
#endif
diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h
index 5a29210dd..a8f58146a 100644
--- a/platform/linux-generic/include/odp_packet_internal.h
+++ b/platform/linux-generic/include/odp_packet_internal.h
@@ -38,15 +38,23 @@ extern "C" {
#include <stdint.h>
#include <string.h>
-/** Minimum segment length expected by _odp_packet_parse_common() */
-#define PACKET_PARSE_SEG_LEN 96
-
ODP_STATIC_ASSERT(sizeof(_odp_packet_input_flags_t) == sizeof(uint64_t),
"INPUT_FLAGS_SIZE_ERROR");
ODP_STATIC_ASSERT(sizeof(_odp_packet_flags_t) == sizeof(uint32_t),
"PACKET_FLAGS_SIZE_ERROR");
+/* Packet extra data length */
+#define PKT_EXTRA_LEN 128
+
+/* Packet extra data types */
+#define PKT_EXTRA_TYPE_DPDK 1
+
+/* Maximum number of segments per packet */
+#define PKT_MAX_SEGS 255
+
+ODP_STATIC_ASSERT(PKT_MAX_SEGS < UINT16_MAX, "PACKET_MAX_SEGS_ERROR");
+
/**
* Packet parser metadata
*/
@@ -67,17 +75,6 @@ typedef struct {
uint16_t l4_offset;
} packet_parser_t;
-/* Packet extra data length */
-#define PKT_EXTRA_LEN 128
-
-/* Packet extra data types */
-#define PKT_EXTRA_TYPE_DPDK 1
-
-/* Maximum number of segments per packet */
-#define PKT_MAX_SEGS 255
-
-ODP_STATIC_ASSERT(PKT_MAX_SEGS < UINT16_MAX, "PACKET_MAX_SEGS_ERROR");
-
/**
* Internal Packet header
*
@@ -146,6 +143,10 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t {
/* Max payload size in a LSO segment */
uint16_t lso_max_payload;
+ /* Packet aging drop timeout before enqueue. Once enqueued holds the maximum age (time of
+ * request + requested drop timeout). */
+ uint64_t tx_aging_ns;
+
/* LSO profile index */
uint8_t lso_profile_idx;
@@ -194,6 +195,11 @@ static inline odp_packet_t packet_from_event_hdr(_odp_event_hdr_t *event_hdr)
return (odp_packet_t)(uintptr_t)event_hdr;
}
+static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr)
+{
+ return pkt_hdr->seg_len;
+}
+
static inline odp_packet_hdr_t *packet_last_seg(odp_packet_hdr_t *hdr)
{
while (hdr->seg_next != NULL)
@@ -400,11 +406,6 @@ static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len)
int _odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len,
odp_packet_t pkt[], int max_num);
-/* Perform packet parse up to a given protocol layer */
-int _odp_packet_parse_layer(odp_packet_hdr_t *pkt_hdr,
- odp_proto_layer_t layer,
- odp_proto_chksums_t chksums);
-
/* Reset parser metadata for a new parse */
static inline void packet_parse_reset(odp_packet_hdr_t *pkt_hdr, int all)
{
@@ -454,12 +455,6 @@ static inline void packet_set_ts(odp_packet_hdr_t *pkt_hdr, odp_time_t *ts)
}
}
-int _odp_packet_parse_common(packet_parser_t *pkt_hdr, const uint8_t *ptr,
- uint32_t pkt_len, uint32_t seg_len, int layer,
- odp_proto_chksums_t chksums);
-
-int _odp_cls_parse(odp_packet_hdr_t *pkt_hdr, const uint8_t *parseptr);
-
int _odp_packet_set_data(odp_packet_t pkt, uint32_t offset,
uint8_t c, uint32_t len);
@@ -471,6 +466,9 @@ int _odp_packet_tcp_chksum_insert(odp_packet_t pkt);
int _odp_packet_udp_chksum_insert(odp_packet_t pkt);
int _odp_packet_sctp_chksum_insert(odp_packet_t pkt);
+int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
+ odp_proto_chksums_t chksums, uint64_t l4_part_sum);
+
#ifdef __cplusplus
}
#endif
diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h
index 3e9f94100..aed6de412 100644
--- a/platform/linux-generic/include/odp_packet_io_internal.h
+++ b/platform/linux-generic/include/odp_packet_io_internal.h
@@ -78,6 +78,7 @@ struct pktio_entry {
/* These two locks together lock the whole pktio device */
odp_ticketlock_t rxl; /**< RX ticketlock */
odp_ticketlock_t txl; /**< TX ticketlock */
+ odp_proto_layer_t parse_layer;
uint16_t pktin_frame_offset;
struct {
@@ -93,6 +94,8 @@ struct pktio_entry {
uint8_t tx_ts : 1;
/* Tx completion events */
uint8_t tx_compl : 1;
+ /* Packet aging */
+ uint8_t tx_aging : 1;
};
};
} enabled;
@@ -144,8 +147,8 @@ struct pktio_entry {
/* Storage for queue handles
* Multi-queue support is pktio driver specific */
- unsigned num_in_queue;
- unsigned num_out_queue;
+ uint32_t num_in_queue;
+ uint32_t num_out_queue;
struct {
odp_queue_t queue;
@@ -227,8 +230,8 @@ typedef struct pktio_if_ops {
int num);
int (*recv_tmo)(pktio_entry_t *entry, int index, odp_packet_t packets[],
int num, uint64_t wait_usecs);
- int (*recv_mq_tmo)(pktio_entry_t *entry[], int index[], int num_q,
- odp_packet_t packets[], int num, unsigned *from,
+ int (*recv_mq_tmo)(pktio_entry_t *entry[], int index[], uint32_t num_q,
+ odp_packet_t packets[], int num, uint32_t *from,
uint64_t wait_usecs);
int (*fd_set)(pktio_entry_t *entry, int index, fd_set *readfds);
int (*send)(pktio_entry_t *entry, int index,
@@ -291,6 +294,11 @@ static inline int _odp_pktio_tx_compl_enabled(const pktio_entry_t *entry)
return entry->s.enabled.tx_compl;
}
+static inline int _odp_pktio_tx_aging_enabled(pktio_entry_t *entry)
+{
+ return entry->s.enabled.tx_aging;
+}
+
static inline void _odp_pktio_tx_ts_set(pktio_entry_t *entry)
{
odp_time_t ts_val = odp_time_global();
@@ -326,7 +334,7 @@ extern const pktio_if_ops_t * const _odp_pktio_if_ops[];
* @return <0 on failure
*/
int _odp_sock_recv_mq_tmo_try_int_driven(const struct odp_pktin_queue_t queues[],
- unsigned int num_q, unsigned int *from,
+ uint32_t num_q, uint32_t *from,
odp_packet_t packets[], int num,
uint64_t usecs,
int *trial_successful);
diff --git a/platform/linux-generic/include/odp_parse_internal.h b/platform/linux-generic/include/odp_parse_internal.h
new file mode 100644
index 000000000..8aa5e118b
--- /dev/null
+++ b/platform/linux-generic/include/odp_parse_internal.h
@@ -0,0 +1,112 @@
+/* Copyright (c) 2013-2018, Linaro Limited
+ * Copyright (c) 2019-2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_PARSE_INTERNAL_H_
+#define ODP_PARSE_INTERNAL_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp_packet_internal.h>
+#include <odp_macros_internal.h>
+#include <protocols/eth.h>
+#include <protocols/ip.h>
+#include <protocols/sctp.h>
+#include <protocols/tcp.h>
+#include <protocols/udp.h>
+#include <odp/api/plat/packet_inline_types.h>
+#include <odp/api/hints.h>
+#include <odp/api/packet_io.h>
+#include <odp/api/packet_types.h>
+#include <stdint.h>
+
+/*
+ * In the worst case we look at the Ethernet header, 8 bytes of LLC/SNAP
+ * header and two VLAN tags in the same packet.
+ */
+#define PARSE_ETH_BYTES (sizeof(_odp_ethhdr_t) + 8 + 2 * sizeof(_odp_vlanhdr_t))
+#define PARSE_IPV4_BYTES (0xfU * 4) /* max IPv4 header length with options */
+/*
+ * Peek 2 bytes beyond IPv6 base header without length check if there are
+ * extension headers.
+ */
+#define PARSE_IPV6_BYTES (sizeof(_odp_ipv6hdr_t) + 2)
+#define PARSE_TCP_BYTES (sizeof(_odp_tcphdr_t))
+/*
+ * In the worst case we look at the UDP header and 4 bytes of the UDP
+ * payload (the non-ESP marker to distinguish IKE packets from ESP packets).
+ */
+#define PARSE_UDP_BYTES (sizeof(_odp_udphdr_t) + 4)
+#define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t))
+
+/* _odp_packet_parse_common_l3_l4() requires up to this many bytes. */
+#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \
+ MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES))
+
+/* _odp_packet_parse_common() requires up to this many bytes. */
+#define PARSE_BYTES (PARSE_ETH_BYTES + PARSE_L3_L4_BYTES)
+
+uint16_t _odp_parse_eth(packet_parser_t *prs, const uint8_t **parseptr,
+ uint32_t *offset, uint32_t frame_len);
+
+/*
+ * Parse common L3 and L4 packet headers up to given layer
+ *
+ * See _odp_packet_parse_common(). Requires up to PARSE_L3_L4_BYTES bytes of
+ * contiguous packet data.
+ */
+int _odp_packet_parse_common_l3_l4(packet_parser_t *prs,
+ const uint8_t *parseptr, uint32_t offset,
+ uint32_t frame_len, uint32_t seg_len,
+ int layer, uint16_t ethtype,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum,
+ odp_pktin_config_opt_t opt);
+
+/**
+ * Parse common packet headers up to given layer
+ *
+ * Requires up to PARSE_BYTES bytes of contiguous packet data. Also parse
+ * metadata must be already initialized.
+ *
+ * Returns 0 on success, 1 on packet errors, and -1 if the packet should be
+ * dropped.
+ */
+static inline int _odp_packet_parse_common(packet_parser_t *prs,
+ const uint8_t *ptr,
+ uint32_t frame_len, uint32_t seg_len,
+ int layer,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum,
+ odp_pktin_config_opt_t opt)
+{
+ uint32_t offset;
+ uint16_t ethtype;
+ const uint8_t *parseptr;
+
+ parseptr = ptr;
+ offset = 0;
+
+ if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE))
+ return 0;
+
+ /* Assume valid L2 header, no CRC/FCS check in SW */
+ prs->l2_offset = offset;
+
+ ethtype = _odp_parse_eth(prs, &parseptr, &offset, frame_len);
+
+ return _odp_packet_parse_common_l3_l4(prs, parseptr, offset, frame_len,
+ seg_len, layer, ethtype, chksums,
+ l4_part_sum, opt);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h
index ba4ad53e8..cec9c8bb8 100644
--- a/platform/linux-generic/include/odp_schedule_if.h
+++ b/platform/linux-generic/include/odp_schedule_if.h
@@ -89,8 +89,6 @@ extern const schedule_fn_t *_odp_sched_fn;
/* Interface for the scheduler */
int _odp_sched_cb_pktin_poll(int pktio_index, int pktin_index,
_odp_event_hdr_t *hdr_tbl[], int num);
-/* evts must have at least QUEUE_MULTI_MAX elements */
-int _odp_sched_cb_pktin_poll_one(int pktio_index, int rx_queue, odp_event_t evts[]);
void _odp_sched_cb_pktio_stop_finalize(int pktio_index);
/* API functions */
diff --git a/platform/linux-generic/include/odp_types_internal.h b/platform/linux-generic/include/odp_types_internal.h
new file mode 100644
index 000000000..a97ac9cd4
--- /dev/null
+++ b/platform/linux-generic/include/odp_types_internal.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_TYPES_INTERNAL_H_
+#define ODP_TYPES_INTERNAL_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef __SIZEOF_INT128__
+
+__extension__ typedef unsigned __int128 _odp_u128_t;
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c
index 3b50232d6..2fdecbc51 100644
--- a/platform/linux-generic/odp_classification.c
+++ b/platform/linux-generic/odp_classification.c
@@ -150,7 +150,7 @@ void odp_cls_pmr_param_init(odp_pmr_param_t *param)
int odp_cls_capability(odp_cls_capability_t *capability)
{
- unsigned int count = 0;
+ uint32_t count = 0;
memset(capability, 0, sizeof(odp_cls_capability_t));
@@ -161,6 +161,7 @@ int odp_cls_capability(odp_cls_capability_t *capability)
capability->max_pmr_terms = CLS_PMR_MAX_ENTRY;
capability->available_pmr_terms = count;
capability->max_cos = CLS_COS_MAX_ENTRY;
+ capability->max_cos_stats = capability->max_cos;
capability->pmr_range_supported = false;
capability->supported_terms.all_bits = 0;
capability->supported_terms.bit.len = 1;
@@ -186,6 +187,8 @@ int odp_cls_capability(odp_cls_capability_t *capability)
capability->threshold_bp.all_bits = 0;
capability->max_hash_queues = CLS_COS_QUEUE_MAX;
capability->max_mark = MAX_MARK;
+ capability->stats.cos.counter.discards = 1;
+ capability->stats.cos.counter.packets = 1;
capability->stats.queue.counter.discards = 1;
capability->stats.queue.counter.packets = 1;
@@ -220,21 +223,29 @@ static inline void _cls_queue_unwind(uint32_t tbl_index, uint32_t j)
odp_queue_destroy(queue_grp_tbl->s.queue[tbl_index + --j]);
}
-odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param)
+odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param_in)
{
uint32_t i, j;
odp_queue_t queue;
odp_cls_drop_t drop_policy;
cos_t *cos;
uint32_t tbl_index;
+ odp_cls_cos_param_t param = *param_in;
+
+ if (param.action == ODP_COS_ACTION_DROP) {
+ param.num_queue = 1;
+ param.queue = ODP_QUEUE_INVALID;
+ param.pool = ODP_POOL_INVALID;
+ param.vector.enable = false;
+ }
/* num_queue should not be zero */
- if (param->num_queue > CLS_COS_QUEUE_MAX || param->num_queue < 1)
+ if (param.num_queue > CLS_COS_QUEUE_MAX || param.num_queue < 1)
return ODP_COS_INVALID;
/* Validate packet vector parameters */
- if (param->vector.enable) {
- odp_pool_t pool = param->vector.pool;
+ if (param.vector.enable) {
+ odp_pool_t pool = param.vector.pool;
odp_pool_info_t pool_info;
if (pool == ODP_POOL_INVALID || odp_pool_info(pool, &pool_info)) {
@@ -245,17 +256,17 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param)
ODP_ERR("wrong pool type\n");
return ODP_COS_INVALID;
}
- if (param->vector.max_size == 0) {
+ if (param.vector.max_size == 0) {
ODP_ERR("vector.max_size is zero\n");
return ODP_COS_INVALID;
}
- if (param->vector.max_size > pool_info.params.vector.max_size) {
+ if (param.vector.max_size > pool_info.params.vector.max_size) {
ODP_ERR("vector.max_size larger than pool max vector size\n");
return ODP_COS_INVALID;
}
}
- drop_policy = param->drop_policy;
+ drop_policy = param.drop_policy;
for (i = 0; i < CLS_COS_MAX_ENTRY; i++) {
cos = &cos_tbl->cos_entry[i];
@@ -274,16 +285,16 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param)
cos->s.linked_cos[j] = NULL;
}
- cos->s.num_queue = param->num_queue;
+ cos->s.num_queue = param.num_queue;
- if (param->num_queue > 1) {
+ if (param.num_queue > 1) {
odp_queue_param_init(&cos->s.queue_param);
cos->s.queue_group = true;
cos->s.queue = ODP_QUEUE_INVALID;
_odp_cls_update_hash_proto(cos,
- param->hash_proto);
+ param.hash_proto);
tbl_index = i * CLS_COS_QUEUE_MAX;
- for (j = 0; j < param->num_queue; j++) {
+ for (j = 0; j < param.num_queue; j++) {
queue = odp_queue_create(NULL, &cos->s.queue_param);
if (queue == ODP_QUEUE_INVALID) {
/* unwind the queues */
@@ -296,21 +307,27 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param)
}
} else {
- cos->s.queue = param->queue;
+ cos->s.queue = param.queue;
}
+
+ odp_atomic_init_u64(&cos->s.stats.discards, 0);
+ odp_atomic_init_u64(&cos->s.stats.packets, 0);
+
/* Initialize statistics counters */
for (j = 0; j < cos->s.num_queue; j++) {
- odp_atomic_init_u64(&cos->s.stats[j].discards, 0);
- odp_atomic_init_u64(&cos->s.stats[j].packets, 0);
+ odp_atomic_init_u64(&cos->s.queue_stats[j].discards, 0);
+ odp_atomic_init_u64(&cos->s.queue_stats[j].packets, 0);
}
- cos->s.pool = param->pool;
+ cos->s.action = param.action;
+ cos->s.pool = param.pool;
cos->s.headroom = 0;
cos->s.valid = 1;
cos->s.drop_policy = drop_policy;
odp_atomic_init_u32(&cos->s.num_rule, 0);
cos->s.index = i;
- cos->s.vector = param->vector;
+ cos->s.vector = param.vector;
+ cos->s.stats_enable = param.stats_enable;
UNLOCK(&cos->s.lock);
return _odp_cos_from_ndx(i);
}
@@ -1461,6 +1478,9 @@ static cos_t *match_pmr_cos(cos_t *cos, const uint8_t *pkt_addr, pmr_t *pmr,
/* PMR matched */
pmr_debug_print(pmr, cos);
+ if (cos->s.stats_enable)
+ odp_atomic_inc_u64(&cos->s.stats.packets);
+
hdr->p.input_flags.cls_mark = 0;
if (pmr->s.mark) {
hdr->p.input_flags.cls_mark = 1;
@@ -1537,8 +1557,11 @@ static inline cos_t *cls_select_cos(pktio_entry_t *entry,
default_cos = cls->default_cos;
/* Return error cos for error packet */
- if (pkt_hdr->p.flags.all.error)
- return cls->error_cos;
+ if (pkt_hdr->p.flags.all.error) {
+ cos = cls->error_cos;
+ goto done;
+ }
+
/* Calls all the PMRs attached at the PKTIO level*/
for (i = 0; i < odp_atomic_load_u32(&default_cos->s.num_rule); i++) {
pmr = default_cos->s.pmr[i];
@@ -1551,11 +1574,17 @@ static inline cos_t *cls_select_cos(pktio_entry_t *entry,
cos = match_qos_cos(entry, pkt_addr, pkt_hdr);
if (cos) {
ODP_DBG_RAW(CLS_DBG, " QoS matched -> cos: %s(%u)\n", cos->s.name, cos->s.index);
- return cos;
+ goto done;
}
ODP_DBG_RAW(CLS_DBG, " No match -> default cos\n");
- return cls->default_cos;
+ cos = cls->default_cos;
+
+done:
+ if (cos && cos->s.stats_enable)
+ odp_atomic_inc_u64(&cos->s.stats.packets);
+
+ return cos;
}
static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr,
@@ -1575,12 +1604,12 @@ static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr,
* @retval 0 on success
* @retval -EFAULT Bug
* @retval -EINVAL Config error
+ * @retval -ENOENT Drop action
*
* @note *base is not released
*/
int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base,
- uint16_t pkt_len, uint32_t seg_len, odp_pool_t *pool,
- odp_packet_hdr_t *pkt_hdr, odp_bool_t parse)
+ odp_pool_t *pool, odp_packet_hdr_t *pkt_hdr)
{
cos_t *cos;
uint32_t tbl_index;
@@ -1588,24 +1617,19 @@ int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base,
ODP_DBG_LVL(CLS_DBG, "Classify packet from %s\n", entry->s.full_name);
- if (parse) {
- packet_parse_reset(pkt_hdr, 1);
- packet_set_len(pkt_hdr, pkt_len);
-
- _odp_packet_parse_common(&pkt_hdr->p, base, pkt_len, seg_len,
- ODP_PROTO_LAYER_ALL,
- entry->s.in_chksums);
- }
cos = cls_select_cos(entry, base, pkt_hdr);
if (cos == NULL)
return -EINVAL;
+ if (cos->s.action == ODP_COS_ACTION_DROP)
+ return -ENOENT;
+
if (cos->s.queue == ODP_QUEUE_INVALID && cos->s.num_queue == 1)
- return -EFAULT;
+ goto error;
if (cos->s.pool == ODP_POOL_INVALID)
- return -EFAULT;
+ goto error;
*pool = cos->s.pool;
pkt_hdr->p.input_flags.dst_queue = 1;
@@ -1623,6 +1647,10 @@ int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base,
cos->s.num_queue);
pkt_hdr->dst_queue = queue_grp_tbl->s.queue[tbl_index];
return 0;
+
+error:
+ odp_atomic_inc_u64(&cos->s.stats.discards);
+ return -EFAULT;
}
static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr,
@@ -1780,6 +1808,27 @@ uint64_t odp_pmr_to_u64(odp_pmr_t hdl)
return _odp_pri(hdl);
}
+int odp_cls_cos_stats(odp_cos_t hdl, odp_cls_cos_stats_t *stats)
+{
+ cos_t *cos = get_cos_entry(hdl);
+
+ if (odp_unlikely(cos == NULL)) {
+ ODP_ERR("Invalid odp_cos_t handle\n");
+ return -1;
+ }
+
+ if (odp_unlikely(stats == NULL)) {
+ ODP_ERR("Output structure NULL\n");
+ return -1;
+ }
+
+ memset(stats, 0, sizeof(*stats));
+ stats->discards = odp_atomic_load_u64(&cos->s.stats.discards);
+ stats->packets = odp_atomic_load_u64(&cos->s.stats.packets);
+
+ return 0;
+}
+
int odp_cls_queue_stats(odp_cos_t hdl, odp_queue_t queue,
odp_cls_queue_stats_t *stats)
{
@@ -1803,8 +1852,8 @@ int odp_cls_queue_stats(odp_cos_t hdl, odp_queue_t queue,
}
memset(stats, 0, sizeof(odp_cls_queue_stats_t));
- stats->discards = odp_atomic_load_u64(&cos->s.stats[queue_idx].discards);
- stats->packets = odp_atomic_load_u64(&cos->s.stats[queue_idx].packets);
+ stats->discards = odp_atomic_load_u64(&cos->s.queue_stats[queue_idx].discards);
+ stats->packets = odp_atomic_load_u64(&cos->s.queue_stats[queue_idx].packets);
return 0;
}
diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c
index bc14bad9a..f276d4659 100644
--- a/platform/linux-generic/odp_crypto_null.c
+++ b/platform/linux-generic/odp_crypto_null.c
@@ -269,9 +269,7 @@ int odp_crypto_session_destroy(odp_crypto_session_t session)
return 0;
}
-/*
- * Shim function around packet operation, can be used by other implementations.
- */
+#if ODP_DEPRECATED_API
int
odp_crypto_operation(odp_crypto_op_param_t *param,
odp_bool_t *posted,
@@ -326,6 +324,7 @@ odp_crypto_operation(odp_crypto_op_param_t *param,
return 0;
}
+#endif
int
_odp_crypto_init_global(void)
@@ -402,6 +401,7 @@ int _odp_crypto_term_local(void)
return 0;
}
+#if ODP_DEPRECATED_API
odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev)
{
/* This check not mandated by the API specification */
@@ -438,6 +438,7 @@ uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl)
{
return _odp_pri(hdl);
}
+#endif
void odp_crypto_session_param_init(odp_crypto_session_param_t *param)
{
diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c
index 13101fbca..6ff0ac041 100644
--- a/platform/linux-generic/odp_crypto_openssl.c
+++ b/platform/linux-generic/odp_crypto_openssl.c
@@ -31,6 +31,7 @@
#include <openssl/hmac.h>
#include <openssl/cmac.h>
#include <openssl/evp.h>
+#include <openssl/opensslv.h>
#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) && !defined(OPENSSL_NO_POLY1305)
#define _ODP_HAVE_CHACHA20_POLY1305 1
@@ -38,6 +39,11 @@
#define _ODP_HAVE_CHACHA20_POLY1305 0
#endif
+/* Ignore warnings about APIs deprecated in OpenSSL 3.0 */
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#endif
+
#define MAX_SESSIONS 4000
#define AES_BLOCK_SIZE 16
#define AES_KEY_LENGTH 16
@@ -2495,6 +2501,7 @@ int odp_crypto_session_destroy(odp_crypto_session_t session)
/*
* Shim function around packet operation, can be used by other implementations.
*/
+#if ODP_DEPRECATED_API
int
odp_crypto_operation(odp_crypto_op_param_t *param,
odp_bool_t *posted,
@@ -2549,6 +2556,7 @@ odp_crypto_operation(odp_crypto_op_param_t *param,
return 0;
}
+#endif
#if OPENSSL_VERSION_NUMBER < 0x10100000L
static void ODP_UNUSED openssl_thread_id(CRYPTO_THREADID ODP_UNUSED *id)
@@ -2705,6 +2713,7 @@ int _odp_crypto_term_local(void)
return 0;
}
+#if ODP_DEPRECATED_API
odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev)
{
/* This check not mandated by the API specification */
@@ -2741,6 +2750,7 @@ uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl)
{
return _odp_pri(hdl);
}
+#endif /* ODP_DEPRECATED_API */
void odp_crypto_session_param_init(odp_crypto_session_param_t *param)
{
diff --git a/platform/linux-generic/odp_dma.c b/platform/linux-generic/odp_dma.c
index 87e3b5b73..4c129c616 100644
--- a/platform/linux-generic/odp_dma.c
+++ b/platform/linux-generic/odp_dma.c
@@ -10,6 +10,7 @@
#include <odp/api/align.h>
#include <odp/api/buffer.h>
#include <odp/api/stash.h>
+#include <odp/api/packet.h>
#include <odp/api/pool.h>
#include <odp/api/queue.h>
diff --git a/platform/linux-generic/odp_event.c b/platform/linux-generic/odp_event.c
index 95be9dec2..90c3e3a3c 100644
--- a/platform/linux-generic/odp_event.c
+++ b/platform/linux-generic/odp_event.c
@@ -24,6 +24,7 @@
#include <odp/api/plat/event_inlines.h>
#include <odp/api/plat/packet_inlines.h>
#include <odp/api/plat/packet_vector_inlines.h>
+#include <odp/api/plat/timer_inlines.h>
odp_event_subtype_t odp_event_subtype(odp_event_t event)
{
@@ -70,9 +71,11 @@ void odp_event_free(odp_event_t event)
case ODP_EVENT_TIMEOUT:
odp_timeout_free(odp_timeout_from_event(event));
break;
+#if ODP_DEPRECATED_API
case ODP_EVENT_CRYPTO_COMPL:
odp_crypto_compl_free(odp_crypto_compl_from_event(event));
break;
+#endif
case ODP_EVENT_IPSEC_STATUS:
_odp_ipsec_status_free(_odp_ipsec_status_from_event(event));
break;
@@ -120,8 +123,10 @@ int odp_event_is_valid(odp_event_t event)
/* Fall through */
case ODP_EVENT_TIMEOUT:
/* Fall through */
+#if ODP_DEPRECATED_API
case ODP_EVENT_CRYPTO_COMPL:
/* Fall through */
+#endif
case ODP_EVENT_IPSEC_STATUS:
/* Fall through */
case ODP_EVENT_PACKET_VECTOR:
diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c
index 4dec8f440..29f0e4bcd 100644
--- a/platform/linux-generic/odp_ipsec.c
+++ b/platform/linux-generic/odp_ipsec.c
@@ -1803,6 +1803,7 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt,
state.out_tunnel.ip_tos = 0;
state.out_tunnel.ip_df = 0;
state.out_tunnel.ip_flabel = 0;
+ state.ip_next_hdr = _ODP_IPPROTO_NO_NEXT;
rc = 0;
} else {
rc = -1;
@@ -2269,12 +2270,6 @@ int odp_ipsec_out_inline(const odp_packet_t pkt_in[], int num_in,
ptr) < 0)
status.error.alg = 1;
- packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC);
- result = ipsec_pkt_result(pkt);
- memset(result, 0, sizeof(*result));
- result->sa = ipsec_sa->ipsec_sa_hdl;
- result->status = status;
-
if (!status.error.all) {
odp_pktout_queue_t pkqueue;
diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c
index c47e4912e..4e4057ede 100644
--- a/platform/linux-generic/odp_ipsec_sad.c
+++ b/platform/linux-generic/odp_ipsec_sad.c
@@ -863,7 +863,10 @@ ipsec_sa_t *_odp_ipsec_sa_lookup(const ipsec_sa_lookup_t *lookup)
lookup->proto == ipsec_sa->proto &&
lookup->spi == ipsec_sa->spi &&
lookup->ver == ipsec_sa->in.lookup_ver &&
- !memcmp(lookup->dst_addr, &ipsec_sa->in.lookup_dst_ipv4,
+ !memcmp(lookup->dst_addr,
+ lookup->ver == ODP_IPSEC_IPV4 ?
+ (void *)&ipsec_sa->in.lookup_dst_ipv4 :
+ (void *)&ipsec_sa->in.lookup_dst_ipv6,
lookup->ver == ODP_IPSEC_IPV4 ?
_ODP_IPV4ADDR_LEN :
_ODP_IPV6ADDR_LEN)) {
diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c
index b91f8ca70..b0d0fc03c 100644
--- a/platform/linux-generic/odp_packet.c
+++ b/platform/linux-generic/odp_packet.c
@@ -16,6 +16,7 @@
#include <odp/api/proto_stats.h>
#include <odp/api/timer.h>
+#include <odp_parse_internal.h>
#include <odp_chksum_internal.h>
#include <odp_debug_internal.h>
#include <odp_errno_define.h>
@@ -131,11 +132,6 @@ static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr,
return pkt_hdr->seg_len;
}
-static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr)
-{
- return pkt_hdr->seg_len;
-}
-
static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr)
{
odp_packet_hdr_t *last_seg = packet_last_seg(pkt_hdr);
@@ -1771,500 +1767,6 @@ static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr,
return sum;
}
-/*
- * In the worst case we look at the Ethernet header, 8 bytes of LLC/SNAP
- * header and two VLAN tags in the same packet.
- */
-#define PARSE_ETH_BYTES (sizeof(_odp_ethhdr_t) + 8 + 2 * sizeof(_odp_vlanhdr_t))
-/** Parser helper function for Ethernet packets
- *
- * Requires up to PARSE_ETH_BYTES bytes of contiguous packet data.
- */
-static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr,
- uint32_t *offset, uint32_t frame_len)
-{
- uint16_t ethtype;
- const _odp_ethhdr_t *eth;
- uint16_t macaddr0, macaddr2, macaddr4;
- const _odp_vlanhdr_t *vlan;
- _odp_packet_input_flags_t input_flags;
-
- input_flags.all = 0;
- input_flags.l2 = 1;
- input_flags.eth = 1;
-
- eth = (const _odp_ethhdr_t *)*parseptr;
-
- /* Detect jumbo frames */
- if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX))
- input_flags.jumbo = 1;
-
- /* Handle Ethernet broadcast/multicast addresses */
- macaddr0 = odp_be_to_cpu_16(*((const uint16_t *)(const void *)eth));
- if (odp_unlikely((macaddr0 & 0x0100) == 0x0100))
- input_flags.eth_mcast = 1;
-
- if (odp_unlikely(macaddr0 == 0xffff)) {
- macaddr2 =
- odp_be_to_cpu_16(*((const uint16_t *)
- (const void *)eth + 1));
- macaddr4 =
- odp_be_to_cpu_16(*((const uint16_t *)
- (const void *)eth + 2));
-
- if ((macaddr2 == 0xffff) && (macaddr4 == 0xffff))
- input_flags.eth_bcast = 1;
- }
-
- /* Get Ethertype */
- ethtype = odp_be_to_cpu_16(eth->type);
- *offset += sizeof(*eth);
- *parseptr += sizeof(*eth);
-
- /* Check for SNAP vs. DIX */
- if (odp_unlikely(ethtype < _ODP_ETH_LEN_MAX)) {
- input_flags.snap = 1;
- if (ethtype > frame_len - *offset) {
- prs->flags.snap_len_err = 1;
- ethtype = 0;
- goto error;
- }
- ethtype = odp_be_to_cpu_16(*((const uint16_t *)(uintptr_t)
- (*parseptr + 6)));
- *offset += 8;
- *parseptr += 8;
- }
-
- /* Parse the VLAN header(s), if present */
- if (odp_unlikely(ethtype == _ODP_ETHTYPE_VLAN_OUTER)) {
- input_flags.vlan_qinq = 1;
- input_flags.vlan = 1;
-
- vlan = (const _odp_vlanhdr_t *)*parseptr;
- ethtype = odp_be_to_cpu_16(vlan->type);
- *offset += sizeof(_odp_vlanhdr_t);
- *parseptr += sizeof(_odp_vlanhdr_t);
- }
-
- if (ethtype == _ODP_ETHTYPE_VLAN) {
- input_flags.vlan = 1;
- vlan = (const _odp_vlanhdr_t *)*parseptr;
- ethtype = odp_be_to_cpu_16(vlan->type);
- *offset += sizeof(_odp_vlanhdr_t);
- *parseptr += sizeof(_odp_vlanhdr_t);
- }
-
- /*
- * The packet was too short for what we parsed. We just give up
- * entirely without trying to parse what fits in the packet.
- */
- if (odp_unlikely(*offset > frame_len)) {
- input_flags.all = 0;
- input_flags.l2 = 1;
- ethtype = 0;
- }
-
-error:
- prs->input_flags.all |= input_flags.all;
-
- return ethtype;
-}
-
-#define PARSE_IPV4_BYTES (0xfU * 4) /* max IPv4 header length with options */
-/**
- * Parser helper function for IPv4
- *
- * Requires up to PARSE_IPV4_BYTES bytes of contiguous packet data.
- */
-static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr,
- uint32_t *offset, uint32_t frame_len,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- const _odp_ipv4hdr_t *ipv4 = (const _odp_ipv4hdr_t *)*parseptr;
- uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr);
- uint32_t l3_len = odp_be_to_cpu_16(ipv4->tot_len);
- uint16_t frag_offset = odp_be_to_cpu_16(ipv4->frag_offset);
- uint8_t ver = _ODP_IPV4HDR_VER(ipv4->ver_ihl);
- uint8_t ihl = _ODP_IPV4HDR_IHL(ipv4->ver_ihl);
-
- if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN ||
- ver != 4 ||
- sizeof(*ipv4) > frame_len - *offset ||
- (l3_len > frame_len - *offset))) {
- prs->flags.ip_err = 1;
- return 0;
- }
-
- if (chksums.chksum.ipv4) {
- prs->input_flags.l3_chksum_done = 1;
- if (chksum_finalize(chksum_partial(ipv4, ihl * 4, 0)) != 0xffff) {
- prs->flags.ip_err = 1;
- prs->flags.l3_chksum_err = 1;
- return 0;
- }
- }
-
- *offset += ihl * 4;
- *parseptr += ihl * 4;
-
- if (chksums.chksum.udp || chksums.chksum.tcp)
- *l4_part_sum = chksum_partial((const uint8_t *)&ipv4->src_addr,
- 2 * _ODP_IPV4ADDR_LEN, 0);
-
- if (odp_unlikely(ihl > _ODP_IPV4HDR_IHL_MIN))
- prs->input_flags.ipopt = 1;
-
- /* A packet is a fragment if:
- * "more fragments" flag is set (all fragments except the last)
- * OR
- * "fragment offset" field is nonzero (all fragments except the first)
- */
- if (odp_unlikely(_ODP_IPV4HDR_IS_FRAGMENT(frag_offset)))
- prs->input_flags.ipfrag = 1;
-
- /* Handle IPv4 broadcast / multicast */
- if (odp_unlikely(dstaddr == 0xffffffff))
- prs->input_flags.ip_bcast = 1;
-
- if (odp_unlikely((dstaddr >> 28) == 0xe))
- prs->input_flags.ip_mcast = 1;
-
- return ipv4->proto;
-}
-
-/*
- * Peeks 2 bytes beyond IPv6 base header without length check if there
- * are extension headers.
- */
-#define PARSE_IPV6_BYTES (sizeof(_odp_ipv6hdr_t) + 2)
-/**
- * Parser helper function for IPv6
- *
- * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data.
- */
-static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr,
- uint32_t *offset, uint32_t frame_len,
- uint32_t seg_len,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr;
- const _odp_ipv6hdr_ext_t *ipv6ext;
- uint32_t dstaddr0 = odp_be_to_cpu_32(ipv6->dst_addr.u8[0]);
- uint32_t l3_len = odp_be_to_cpu_16(ipv6->payload_len) +
- _ODP_IPV6HDR_LEN;
-
- /* Basic sanity checks on IPv6 header */
- if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 ||
- sizeof(*ipv6) > frame_len - *offset ||
- l3_len > frame_len - *offset)) {
- prs->flags.ip_err = 1;
- return 0;
- }
-
- /* IPv6 broadcast / multicast flags */
- prs->input_flags.ip_mcast = (dstaddr0 & 0xff000000) == 0xff000000;
- prs->input_flags.ip_bcast = 0;
-
- /* Skip past IPv6 header */
- *offset += sizeof(_odp_ipv6hdr_t);
- *parseptr += sizeof(_odp_ipv6hdr_t);
-
- if (chksums.chksum.udp || chksums.chksum.tcp)
- *l4_part_sum = chksum_partial((const uint8_t *)&ipv6->src_addr,
- 2 * _ODP_IPV6ADDR_LEN, 0);
-
- /* Skip past any IPv6 extension headers */
- if (ipv6->next_hdr == _ODP_IPPROTO_HOPOPTS ||
- ipv6->next_hdr == _ODP_IPPROTO_ROUTE) {
- prs->input_flags.ipopt = 1;
-
- do {
- ipv6ext = (const _odp_ipv6hdr_ext_t *)*parseptr;
- uint16_t extlen = 8 + ipv6ext->ext_len * 8;
-
- *offset += extlen;
- *parseptr += extlen;
- } while ((ipv6ext->next_hdr == _ODP_IPPROTO_HOPOPTS ||
- ipv6ext->next_hdr == _ODP_IPPROTO_ROUTE) &&
- *offset < seg_len);
-
- if (*offset >= prs->l3_offset +
- odp_be_to_cpu_16(ipv6->payload_len)) {
- prs->flags.ip_err = 1;
- return 0;
- }
-
- if (ipv6ext->next_hdr == _ODP_IPPROTO_FRAG)
- prs->input_flags.ipfrag = 1;
-
- return ipv6ext->next_hdr;
- }
-
- if (odp_unlikely(ipv6->next_hdr == _ODP_IPPROTO_FRAG)) {
- prs->input_flags.ipopt = 1;
- prs->input_flags.ipfrag = 1;
- }
-
- return ipv6->next_hdr;
-}
-
-#define PARSE_TCP_BYTES (sizeof(_odp_tcphdr_t))
-/**
- * Parser helper function for TCP
- *
- * Requires PARSE_TCP_BYTES bytes of contiguous packet data.
- */
-static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr,
- uint16_t tcp_len,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- const _odp_tcphdr_t *tcp = (const _odp_tcphdr_t *)*parseptr;
- uint32_t len = tcp->hl * 4;
-
- if (odp_unlikely(tcp->hl < sizeof(_odp_tcphdr_t) / sizeof(uint32_t)))
- prs->flags.tcp_err = 1;
-
- if (chksums.chksum.tcp &&
- !prs->input_flags.ipfrag) {
- *l4_part_sum += odp_cpu_to_be_16(tcp_len);
-#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
- *l4_part_sum += _ODP_IPPROTO_TCP;
-#else
- *l4_part_sum += _ODP_IPPROTO_TCP << 8;
-#endif
- }
-
- *parseptr += len;
-}
-
-/*
- * In the worst case we look at the UDP header and 4 bytes of the UDP
- * payload (the non-ESP marker to distinguish IKE packets from ESP packets).
- */
-#define PARSE_UDP_BYTES (sizeof(_odp_udphdr_t) + 4)
-/**
- * Parser helper function for UDP
- *
- * Requires PARSE_UDP_BYTES bytes of contiguous packet data.
- */
-static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr;
- uint32_t udplen = odp_be_to_cpu_16(udp->length);
- uint16_t ipsec_port = odp_cpu_to_be_16(_ODP_UDP_IPSEC_PORT);
-
- if (odp_unlikely(udplen < sizeof(_odp_udphdr_t))) {
- prs->flags.udp_err = 1;
- return;
- }
-
- if (chksums.chksum.udp &&
- !prs->input_flags.ipfrag) {
- if (udp->chksum == 0) {
- prs->input_flags.l4_chksum_done = 1;
- prs->flags.l4_chksum_err =
- (prs->input_flags.ipv4 != 1);
- } else {
- *l4_part_sum += udp->length;
-#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
- *l4_part_sum += _ODP_IPPROTO_UDP;
-#else
- *l4_part_sum += _ODP_IPPROTO_UDP << 8;
-#endif
- }
- prs->input_flags.udp_chksum_zero = (udp->chksum == 0);
- }
-
- if (odp_unlikely(ipsec_port == udp->dst_port && udplen > 4)) {
- uint32_t val;
-
- memcpy(&val, udp + 1, 4);
- if (val != 0) {
- prs->input_flags.ipsec = 1;
- prs->input_flags.ipsec_udp = 1;
- }
- }
-
- *parseptr += sizeof(_odp_udphdr_t);
-}
-
-#define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t))
-/**
- * Parser helper function for SCTP
- *
- * Requires PARSE_SCTP_BYTES bytes of contiguous packet data.
- */
-static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr,
- uint16_t sctp_len,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) {
- prs->flags.sctp_err = 1;
- return;
- }
-
- if (chksums.chksum.sctp &&
- !prs->input_flags.ipfrag) {
- const _odp_sctphdr_t *sctp =
- (const _odp_sctphdr_t *)*parseptr;
- uint32_t crc = ~0;
- uint32_t zero = 0;
-
- crc = odp_hash_crc32c(sctp, sizeof(*sctp) - 4, crc);
- crc = odp_hash_crc32c(&zero, 4, crc);
- *l4_part_sum = crc;
- }
-
- *parseptr += sizeof(_odp_sctphdr_t);
-}
-
-#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c)))
-#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \
- MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES))
-/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */
-static inline
-int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr,
- uint32_t offset,
- uint32_t frame_len, uint32_t seg_len,
- int layer, uint16_t ethtype,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- uint8_t ip_proto;
-
- prs->l3_offset = offset;
-
- if (odp_unlikely(layer <= ODP_PROTO_LAYER_L2))
- return prs->flags.all.error != 0;
-
- /* Set l3 flag only for known ethtypes */
- prs->input_flags.l3 = 1;
-
- /* Parse Layer 3 headers */
- switch (ethtype) {
- case _ODP_ETHTYPE_IPV4:
- prs->input_flags.ipv4 = 1;
- ip_proto = parse_ipv4(prs, &parseptr, &offset, frame_len,
- chksums, l4_part_sum);
- prs->l4_offset = offset;
- break;
-
- case _ODP_ETHTYPE_IPV6:
- prs->input_flags.ipv6 = 1;
- ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len,
- seg_len, chksums, l4_part_sum);
- prs->l4_offset = offset;
- break;
-
- case _ODP_ETHTYPE_ARP:
- prs->input_flags.arp = 1;
- ip_proto = 255; /* Reserved invalid by IANA */
- break;
-
- default:
- prs->input_flags.l3 = 0;
- ip_proto = 255; /* Reserved invalid by IANA */
- }
-
- if (layer == ODP_PROTO_LAYER_L3)
- return prs->flags.all.error != 0;
-
- /* Set l4 flag only for known ip_proto */
- prs->input_flags.l4 = 1;
-
- /* Parse Layer 4 headers */
- switch (ip_proto) {
- case _ODP_IPPROTO_ICMPV4:
- /* Fall through */
-
- case _ODP_IPPROTO_ICMPV6:
- prs->input_flags.icmp = 1;
- break;
-
- case _ODP_IPPROTO_IPIP:
- /* Do nothing */
- break;
-
- case _ODP_IPPROTO_TCP:
- if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len))
- return -1;
- prs->input_flags.tcp = 1;
- parse_tcp(prs, &parseptr, frame_len - prs->l4_offset, chksums,
- l4_part_sum);
- break;
-
- case _ODP_IPPROTO_UDP:
- if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len))
- return -1;
- prs->input_flags.udp = 1;
- parse_udp(prs, &parseptr, chksums, l4_part_sum);
- break;
-
- case _ODP_IPPROTO_AH:
- prs->input_flags.ipsec = 1;
- prs->input_flags.ipsec_ah = 1;
- break;
-
- case _ODP_IPPROTO_ESP:
- prs->input_flags.ipsec = 1;
- prs->input_flags.ipsec_esp = 1;
- break;
-
- case _ODP_IPPROTO_SCTP:
- prs->input_flags.sctp = 1;
- parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, chksums,
- l4_part_sum);
- break;
-
- case _ODP_IPPROTO_NO_NEXT:
- prs->input_flags.no_next_hdr = 1;
- break;
-
- default:
- prs->input_flags.l4 = 0;
- break;
- }
-
- return prs->flags.all.error != 0;
-}
-
-/**
- * Parse common packet headers up to given layer
- *
- * The function expects at least PACKET_PARSE_SEG_LEN bytes of data to be
- * available from the ptr. Also parse metadata must be already initialized.
- */
-int _odp_packet_parse_common(packet_parser_t *prs, const uint8_t *ptr,
- uint32_t frame_len, uint32_t seg_len,
- int layer, odp_proto_chksums_t chksums)
-{
- uint32_t offset;
- uint16_t ethtype;
- const uint8_t *parseptr;
- uint64_t l4_part_sum;
-
- parseptr = ptr;
- offset = 0;
-
- if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE))
- return 0;
-
- /* Assume valid L2 header, no CRC/FCS check in SW */
- prs->l2_offset = offset;
-
- ethtype = parse_eth(prs, &parseptr, &offset, frame_len);
-
- return packet_parse_common_l3_l4(prs, parseptr, offset, frame_len,
- seg_len, layer, ethtype, chksums,
- &l4_part_sum);
-}
-
static inline int packet_ipv4_chksum(odp_packet_t pkt,
uint32_t offset,
_odp_ipv4hdr_t *ip,
@@ -2442,9 +1944,8 @@ int _odp_packet_sctp_chksum_insert(odp_packet_t pkt)
return odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum);
}
-static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
- odp_proto_chksums_t chksums,
- uint64_t l4_part_sum)
+int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
+ odp_proto_chksums_t chksums, uint64_t l4_part_sum)
{
/* UDP chksum == 0 case is covered in parse_udp() */
if (chksums.chksum.udp &&
@@ -2518,42 +2019,6 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
return pkt_hdr->p.flags.all.error != 0;
}
-/**
- * Simple packet parser
- */
-int _odp_packet_parse_layer(odp_packet_hdr_t *pkt_hdr,
- odp_proto_layer_t layer,
- odp_proto_chksums_t chksums)
-{
- uint32_t seg_len = packet_first_seg_len(pkt_hdr);
- const uint8_t *base = packet_data(pkt_hdr);
- uint32_t offset = 0;
- uint16_t ethtype;
- uint64_t l4_part_sum = 0;
- int rc;
-
- if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE))
- return 0;
-
- /* Assume valid L2 header, no CRC/FCS check in SW */
- pkt_hdr->p.l2_offset = offset;
-
- ethtype = parse_eth(&pkt_hdr->p, &base, &offset, pkt_hdr->frame_len);
-
- rc = packet_parse_common_l3_l4(&pkt_hdr->p, base, offset,
- pkt_hdr->frame_len,
- seg_len, layer, ethtype, chksums,
- &l4_part_sum);
-
- if (rc != 0)
- return rc;
-
- if (layer >= ODP_PROTO_LAYER_L4)
- return packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
- else
- return 0;
-}
-
int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
const odp_packet_parse_param_t *param)
{
@@ -2568,6 +2033,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
uint64_t l4_part_sum = 0;
const uint32_t min_seglen = PARSE_ETH_BYTES + PARSE_L3_L4_BYTES;
uint8_t buf[min_seglen];
+ odp_pktin_config_opt_t opt;
if (proto == ODP_PROTO_NONE || layer == ODP_PROTO_LAYER_NONE)
return -1;
@@ -2598,7 +2064,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
/* Assume valid L2 header, no CRC/FCS check in SW */
pkt_hdr->p.l2_offset = offset;
- ethtype = parse_eth(&pkt_hdr->p, &data, &offset, packet_len);
+ ethtype = _odp_parse_eth(&pkt_hdr->p, &data, &offset, packet_len);
} else if (proto == ODP_PROTO_IPV4) {
ethtype = _ODP_ETHTYPE_IPV4;
} else if (proto == ODP_PROTO_IPV6) {
@@ -2607,17 +2073,18 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
ethtype = 0; /* Invalid */
}
- ret = packet_parse_common_l3_l4(&pkt_hdr->p, data, offset,
- packet_len, seg_len,
- layer, ethtype,
- param->chksums,
- &l4_part_sum);
+ opt.all_bits = 0;
+
+ ret = _odp_packet_parse_common_l3_l4(&pkt_hdr->p, data, offset,
+ packet_len, seg_len, layer,
+ ethtype, param->chksums,
+ &l4_part_sum, opt);
if (ret)
return -1;
if (layer >= ODP_PROTO_LAYER_L4) {
- ret = packet_l4_chksum(pkt_hdr, param->chksums, l4_part_sum);
+ ret = _odp_packet_l4_chksum(pkt_hdr, param->chksums, l4_part_sum);
if (ret)
return -1;
}
@@ -2879,14 +2346,17 @@ int odp_packet_payload_offset_set(odp_packet_t pkt, uint32_t offset)
void odp_packet_aging_tmo_set(odp_packet_t pkt, uint64_t tmo_ns)
{
- (void)pkt;
- (void)tmo_ns;
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+
+ pkt_hdr->p.flags.tx_aging = tmo_ns ? 1 : 0;
+ pkt_hdr->tx_aging_ns = tmo_ns;
}
uint64_t odp_packet_aging_tmo(odp_packet_t pkt)
{
- (void)pkt;
- return 0;
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+
+ return pkt_hdr->p.flags.tx_aging ? pkt_hdr->tx_aging_ns : 0;
}
int odp_packet_tx_compl_request(odp_packet_t pkt, const odp_packet_tx_compl_opt_t *opt)
diff --git a/platform/linux-generic/odp_packet_flags.c b/platform/linux-generic/odp_packet_flags.c
index bb5c26792..ef542a2cd 100644
--- a/platform/linux-generic/odp_packet_flags.c
+++ b/platform/linux-generic/odp_packet_flags.c
@@ -8,137 +8,11 @@
#include <odp/api/packet_flags.h>
#include <odp_packet_internal.h>
-#define retflag(pkt, x) do { \
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); \
- return pkt_hdr->p.x; \
- } while (0)
-
#define setflag(pkt, x, v) do { \
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); \
pkt_hdr->p.x = (v) & 1; \
} while (0)
-int odp_packet_has_error(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
-
- return pkt_hdr->p.flags.all.error != 0;
-}
-
-/* Get Input Flags */
-
-int odp_packet_has_l2_error(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
- /* L2 parsing is always done by default and hence
- no additional check is required */
- return pkt_hdr->p.flags.snap_len_err;
-}
-
-int odp_packet_has_l3(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.l3);
-}
-
-int odp_packet_has_l3_error(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
-
- return pkt_hdr->p.flags.ip_err;
-}
-
-int odp_packet_has_l4(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.l4);
-}
-
-int odp_packet_has_l4_error(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
-
- return pkt_hdr->p.flags.tcp_err | pkt_hdr->p.flags.udp_err;
-}
-
-int odp_packet_has_eth_bcast(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.eth_bcast);
-}
-
-int odp_packet_has_eth_mcast(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.eth_mcast);
-}
-
-int odp_packet_has_vlan(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.vlan);
-}
-
-int odp_packet_has_vlan_qinq(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.vlan_qinq);
-}
-
-int odp_packet_has_arp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.arp);
-}
-
-int odp_packet_has_ipv4(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ipv4);
-}
-
-int odp_packet_has_ipv6(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ipv6);
-}
-
-int odp_packet_has_ip_bcast(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ip_bcast);
-}
-
-int odp_packet_has_ip_mcast(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ip_mcast);
-}
-
-int odp_packet_has_ipfrag(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ipfrag);
-}
-
-int odp_packet_has_ipopt(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ipopt);
-}
-
-int odp_packet_has_udp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.udp);
-}
-
-int odp_packet_has_tcp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.tcp);
-}
-
-int odp_packet_has_sctp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.sctp);
-}
-
-int odp_packet_has_icmp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.icmp);
-}
-
-odp_packet_color_t odp_packet_color(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.color);
-}
-
void odp_packet_color_set(odp_packet_t pkt, odp_packet_color_t color)
{
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
@@ -146,23 +20,11 @@ void odp_packet_color_set(odp_packet_t pkt, odp_packet_color_t color)
pkt_hdr->p.input_flags.color = color;
}
-odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
-
- return !pkt_hdr->p.input_flags.nodrop;
-}
-
void odp_packet_drop_eligible_set(odp_packet_t pkt, odp_bool_t drop)
{
setflag(pkt, input_flags.nodrop, !drop);
}
-int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt)
-{
- retflag(pkt, flags.shaper_len_adj);
-}
-
void odp_packet_shaper_len_adjust_set(odp_packet_t pkt, int8_t adj)
{
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c
index 899f10425..7c68b10c5 100644
--- a/platform/linux-generic/odp_packet_io.c
+++ b/platform/linux-generic/odp_packet_io.c
@@ -53,6 +53,10 @@
/* Max wait time supported to avoid potential overflow */
#define MAX_WAIT_TIME (UINT64_MAX / 1024)
+/* One hour maximum aging timeout, no real limitations imposed by the implementation other than
+ * integer width, so just use some value. */
+#define MAX_TX_AGING_TMO_NS 3600000000000ULL
+
typedef struct {
const void *user_ptr;
odp_queue_t queue;
@@ -624,6 +628,8 @@ int odp_pktio_config(odp_pktio_t hdl, const odp_pktio_config_t *config)
return -1;
}
+ entry->s.enabled.tx_aging = config->pktout.bit.aging_ena;
+
if (entry->s.ops->config)
res = entry->s.ops->config(entry, config);
@@ -650,6 +656,9 @@ int odp_pktio_start(odp_pktio_t hdl)
ODP_ERR("Already started\n");
return -1;
}
+ entry->s.parse_layer = pktio_cls_enabled(entry) ?
+ ODP_PROTO_LAYER_ALL :
+ entry->s.config.parser.layer;
if (entry->s.ops->start)
res = entry->s.ops->start(entry);
if (!res)
@@ -660,8 +669,8 @@ int odp_pktio_start(odp_pktio_t hdl)
mode = entry->s.param.in_mode;
if (mode == ODP_PKTIN_MODE_SCHED) {
- unsigned int i;
- unsigned int num = entry->s.num_in_queue;
+ uint32_t i;
+ uint32_t num = entry->s.num_in_queue;
int index[num];
odp_queue_t odpq[num];
@@ -1182,95 +1191,6 @@ static int pktin_deq_multi(odp_queue_t queue, _odp_event_hdr_t *event_hdr[],
return nbr;
}
-int _odp_sched_cb_pktin_poll_one(int pktio_index,
- int rx_queue,
- odp_event_t evt_tbl[])
-{
- int num_rx, num_pkts, i;
- pktio_entry_t *entry = pktio_entry_by_index(pktio_index);
- odp_packet_t pkt;
- odp_packet_hdr_t *pkt_hdr;
- odp_pool_t pool = ODP_POOL_INVALID;
- odp_packet_t packets[QUEUE_MULTI_MAX];
- odp_queue_t queue;
- odp_bool_t vector_enabled = entry->s.in_queue[rx_queue].vector.enable;
- uint32_t num = QUEUE_MULTI_MAX;
- cos_t *cos_hdr = NULL;
-
- if (odp_unlikely(entry->s.state != PKTIO_STATE_STARTED)) {
- if (entry->s.state < PKTIO_STATE_ACTIVE ||
- entry->s.state == PKTIO_STATE_STOP_PENDING)
- return -1;
-
- ODP_DBG("interface not started\n");
- return 0;
- }
-
- if (vector_enabled) {
- /* Make sure all packets will fit into a single packet vector */
- if (entry->s.in_queue[rx_queue].vector.max_size < num)
- num = entry->s.in_queue[rx_queue].vector.max_size;
- pool = entry->s.in_queue[rx_queue].vector.pool;
- }
-
- ODP_ASSERT((unsigned int)rx_queue < entry->s.num_in_queue);
- num_pkts = entry->s.ops->recv(entry, rx_queue, packets, num);
-
- num_rx = 0;
- for (i = 0; i < num_pkts; i++) {
- pkt = packets[i];
- pkt_hdr = packet_hdr(pkt);
- if (odp_unlikely(pkt_hdr->p.input_flags.dst_queue)) {
- odp_event_t event = odp_packet_to_event(pkt);
- uint16_t cos_idx = pkt_hdr->cos;
-
- queue = pkt_hdr->dst_queue;
-
- if (cos_idx != CLS_COS_IDX_NONE) {
- /* Packets from classifier */
- cos_hdr = _odp_cos_entry_from_idx(cos_idx);
-
- if (cos_hdr->s.vector.enable) {
- packet_vector_enq_cos(queue, &event, 1, cos_hdr);
- continue;
- }
- } else if (vector_enabled) {
- /* Packets from inline IPsec */
- packet_vector_enq(queue, &event, 1, pool);
- continue;
- }
-
- if (odp_unlikely(odp_queue_enq(queue, event))) {
- /* Queue full? */
- odp_packet_free(pkt);
- if (cos_idx != CLS_COS_IDX_NONE)
- _odp_cos_queue_stats_add(cos_hdr, queue, 0, 1);
- else
- odp_atomic_inc_u64(&entry->s.stats_extra.in_discards);
- } else {
- if (cos_idx != CLS_COS_IDX_NONE)
- _odp_cos_queue_stats_add(cos_hdr, queue, 1, 0);
- }
- } else {
- evt_tbl[num_rx++] = odp_packet_to_event(pkt);
- }
- }
-
- /* Create packet vector */
- if (vector_enabled && num_rx > 0) {
- odp_packet_vector_t pktv = packet_vector_create((odp_packet_t *)evt_tbl,
- num_rx, pool);
-
- if (odp_unlikely(pktv == ODP_PACKET_VECTOR_INVALID))
- return 0;
-
- evt_tbl[0] = odp_packet_vector_to_event(pktv);
- return 1;
- }
-
- return num_rx;
-}
-
int _odp_sched_cb_pktin_poll(int pktio_index, int pktin_index,
_odp_event_hdr_t *hdr_tbl[], int num)
{
@@ -1886,6 +1806,9 @@ int odp_pktio_capability(odp_pktio_t pktio, odp_pktio_capability_t *capa)
capa->tx_compl.queue_type_sched = 1;
capa->tx_compl.queue_type_plain = 1;
capa->tx_compl.mode_all = 1;
+
+ capa->config.pktout.bit.aging_ena = 1;
+ capa->max_tx_aging_tmo_ns = MAX_TX_AGING_TMO_NS;
}
/* Packet vector generation is common for all pktio types */
@@ -2246,8 +2169,7 @@ int odp_pktin_queue_config(odp_pktio_t pktio,
pktio_entry_t *entry;
odp_pktin_mode_t mode;
odp_pktio_capability_t capa;
- unsigned int num_queues;
- unsigned int i;
+ uint32_t num_queues, i;
int rc;
odp_queue_t queue;
odp_pktin_queue_param_t default_param;
@@ -2410,7 +2332,7 @@ int _odp_pktio_pktout_tm_config(odp_pktio_t pktio_hdl,
bool pktio_started = false;
odp_pktout_mode_t mode;
pktio_entry_t *entry;
- unsigned int i;
+ uint32_t i;
int rc = 0;
odp_pktout_queue_param_init(&param);
@@ -2480,8 +2402,7 @@ int odp_pktout_queue_config(odp_pktio_t pktio,
pktio_entry_t *entry;
odp_pktout_mode_t mode;
odp_pktio_capability_t capa;
- unsigned int num_queues;
- unsigned int i;
+ uint32_t num_queues, i;
int rc;
odp_pktout_queue_param_t default_param;
@@ -2847,18 +2768,17 @@ int odp_pktin_recv_tmo(odp_pktin_queue_t queue, odp_packet_t packets[], int num,
}
}
-int odp_pktin_recv_mq_tmo(const odp_pktin_queue_t queues[], unsigned int num_q,
- unsigned int *from, odp_packet_t packets[], int num,
- uint64_t wait)
+int odp_pktin_recv_mq_tmo(const odp_pktin_queue_t queues[], uint32_t num_q, uint32_t *from,
+ odp_packet_t packets[], int num, uint64_t wait)
{
- unsigned int i;
+ uint32_t i;
int ret;
odp_time_t t1, t2;
struct timespec ts;
int started = 0;
uint64_t sleep_round = 0;
int trial_successful = 0;
- unsigned int lfrom = 0;
+ uint32_t lfrom = 0;
for (i = 0; i < num_q; i++) {
ret = odp_pktin_recv(queues[i], packets, num);
diff --git a/platform/linux-generic/odp_parse.c b/platform/linux-generic/odp_parse.c
new file mode 100644
index 000000000..2342a7e49
--- /dev/null
+++ b/platform/linux-generic/odp_parse.c
@@ -0,0 +1,471 @@
+/* Copyright (c) 2013-2018, Linaro Limited
+ * Copyright (c) 2019-2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp_parse_internal.h>
+#include <odp_chksum_internal.h>
+#include <protocols/eth.h>
+#include <protocols/ip.h>
+#include <protocols/sctp.h>
+#include <protocols/tcp.h>
+#include <protocols/udp.h>
+#include <odp/api/hash.h>
+#include <odp/api/packet_io.h>
+#include <odp/api/packet_types.h>
+#include <stdint.h>
+#include <string.h>
+
+/** Parser helper function for Ethernet packets
+ *
+ * Requires up to PARSE_ETH_BYTES bytes of contiguous packet data.
+ */
+uint16_t _odp_parse_eth(packet_parser_t *prs, const uint8_t **parseptr,
+ uint32_t *offset, uint32_t frame_len)
+{
+ uint16_t ethtype;
+ const _odp_ethhdr_t *eth;
+ uint16_t macaddr0, macaddr2, macaddr4;
+ const _odp_vlanhdr_t *vlan;
+ _odp_packet_input_flags_t input_flags;
+
+ input_flags.all = 0;
+ input_flags.l2 = 1;
+ input_flags.eth = 1;
+
+ eth = (const _odp_ethhdr_t *)*parseptr;
+
+ /* Detect jumbo frames */
+ if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX))
+ input_flags.jumbo = 1;
+
+ /* Handle Ethernet broadcast/multicast addresses */
+ macaddr0 = odp_be_to_cpu_16(*((const uint16_t *)(const void *)eth));
+ if (odp_unlikely((macaddr0 & 0x0100) == 0x0100))
+ input_flags.eth_mcast = 1;
+
+ if (odp_unlikely(macaddr0 == 0xffff)) {
+ macaddr2 =
+ odp_be_to_cpu_16(*((const uint16_t *)
+ (const void *)eth + 1));
+ macaddr4 =
+ odp_be_to_cpu_16(*((const uint16_t *)
+ (const void *)eth + 2));
+
+ if ((macaddr2 == 0xffff) && (macaddr4 == 0xffff))
+ input_flags.eth_bcast = 1;
+ }
+
+ /* Get Ethertype */
+ ethtype = odp_be_to_cpu_16(eth->type);
+ *offset += sizeof(*eth);
+ *parseptr += sizeof(*eth);
+
+ /* Check for SNAP vs. DIX */
+ if (odp_unlikely(ethtype < _ODP_ETH_LEN_MAX)) {
+ input_flags.snap = 1;
+ if (ethtype > frame_len - *offset) {
+ prs->flags.snap_len_err = 1;
+ ethtype = 0;
+ goto error;
+ }
+ ethtype = odp_be_to_cpu_16(*((const uint16_t *)(uintptr_t)
+ (*parseptr + 6)));
+ *offset += 8;
+ *parseptr += 8;
+ }
+
+ /* Parse the VLAN header(s), if present */
+ if (odp_unlikely(ethtype == _ODP_ETHTYPE_VLAN_OUTER)) {
+ input_flags.vlan_qinq = 1;
+ input_flags.vlan = 1;
+
+ vlan = (const _odp_vlanhdr_t *)*parseptr;
+ ethtype = odp_be_to_cpu_16(vlan->type);
+ *offset += sizeof(_odp_vlanhdr_t);
+ *parseptr += sizeof(_odp_vlanhdr_t);
+ }
+
+ if (ethtype == _ODP_ETHTYPE_VLAN) {
+ input_flags.vlan = 1;
+ vlan = (const _odp_vlanhdr_t *)*parseptr;
+ ethtype = odp_be_to_cpu_16(vlan->type);
+ *offset += sizeof(_odp_vlanhdr_t);
+ *parseptr += sizeof(_odp_vlanhdr_t);
+ }
+
+ /*
+ * The packet was too short for what we parsed. We just give up
+ * entirely without trying to parse what fits in the packet.
+ */
+ if (odp_unlikely(*offset > frame_len)) {
+ input_flags.all = 0;
+ input_flags.l2 = 1;
+ ethtype = 0;
+ }
+
+error:
+ prs->input_flags.all |= input_flags.all;
+
+ return ethtype;
+}
+
+/**
+ * Parser helper function for IPv4
+ *
+ * Requires up to PARSE_IPV4_BYTES bytes of contiguous packet data.
+ */
+static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr,
+ uint32_t *offset, uint32_t frame_len,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum)
+{
+ const _odp_ipv4hdr_t *ipv4 = (const _odp_ipv4hdr_t *)*parseptr;
+ uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr);
+ uint32_t l3_len = odp_be_to_cpu_16(ipv4->tot_len);
+ uint16_t frag_offset = odp_be_to_cpu_16(ipv4->frag_offset);
+ uint8_t ver = _ODP_IPV4HDR_VER(ipv4->ver_ihl);
+ uint8_t ihl = _ODP_IPV4HDR_IHL(ipv4->ver_ihl);
+
+ if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN ||
+ ver != 4 ||
+ sizeof(*ipv4) > frame_len - *offset ||
+ (l3_len > frame_len - *offset))) {
+ prs->flags.ip_err = 1;
+ return 0;
+ }
+
+ if (chksums.chksum.ipv4) {
+ prs->input_flags.l3_chksum_done = 1;
+ if (chksum_finalize(chksum_partial(ipv4, ihl * 4, 0)) != 0xffff) {
+ prs->flags.ip_err = 1;
+ prs->flags.l3_chksum_err = 1;
+ return 0;
+ }
+ }
+
+ *offset += ihl * 4;
+ *parseptr += ihl * 4;
+
+ if (chksums.chksum.udp || chksums.chksum.tcp)
+ *l4_part_sum = chksum_partial((const uint8_t *)&ipv4->src_addr,
+ 2 * _ODP_IPV4ADDR_LEN, 0);
+
+ if (odp_unlikely(ihl > _ODP_IPV4HDR_IHL_MIN))
+ prs->input_flags.ipopt = 1;
+
+ /* A packet is a fragment if:
+ * "more fragments" flag is set (all fragments except the last)
+ * OR
+ * "fragment offset" field is nonzero (all fragments except the first)
+ */
+ if (odp_unlikely(_ODP_IPV4HDR_IS_FRAGMENT(frag_offset)))
+ prs->input_flags.ipfrag = 1;
+
+ /* Handle IPv4 broadcast / multicast */
+ if (odp_unlikely(dstaddr == 0xffffffff))
+ prs->input_flags.ip_bcast = 1;
+
+ if (odp_unlikely((dstaddr >> 28) == 0xe))
+ prs->input_flags.ip_mcast = 1;
+
+ return ipv4->proto;
+}
+
+/**
+ * Parser helper function for IPv6
+ *
+ * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data.
+ */
+static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr,
+ uint32_t *offset, uint32_t frame_len,
+ uint32_t seg_len,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum)
+{
+ const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr;
+ const _odp_ipv6hdr_ext_t *ipv6ext;
+ uint32_t dstaddr0 = odp_be_to_cpu_32(ipv6->dst_addr.u8[0]);
+ uint32_t l3_len = odp_be_to_cpu_16(ipv6->payload_len) +
+ _ODP_IPV6HDR_LEN;
+
+ /* Basic sanity checks on IPv6 header */
+ if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 ||
+ sizeof(*ipv6) > frame_len - *offset ||
+ l3_len > frame_len - *offset)) {
+ prs->flags.ip_err = 1;
+ return 0;
+ }
+
+ /* IPv6 broadcast / multicast flags */
+ prs->input_flags.ip_mcast = (dstaddr0 & 0xff000000) == 0xff000000;
+ prs->input_flags.ip_bcast = 0;
+
+ /* Skip past IPv6 header */
+ *offset += sizeof(_odp_ipv6hdr_t);
+ *parseptr += sizeof(_odp_ipv6hdr_t);
+
+ if (chksums.chksum.udp || chksums.chksum.tcp)
+ *l4_part_sum = chksum_partial((const uint8_t *)&ipv6->src_addr,
+ 2 * _ODP_IPV6ADDR_LEN, 0);
+
+ /* Skip past any IPv6 extension headers */
+ if (ipv6->next_hdr == _ODP_IPPROTO_HOPOPTS ||
+ ipv6->next_hdr == _ODP_IPPROTO_ROUTE) {
+ prs->input_flags.ipopt = 1;
+
+ do {
+ ipv6ext = (const _odp_ipv6hdr_ext_t *)*parseptr;
+ uint16_t extlen = 8 + ipv6ext->ext_len * 8;
+
+ *offset += extlen;
+ *parseptr += extlen;
+ } while ((ipv6ext->next_hdr == _ODP_IPPROTO_HOPOPTS ||
+ ipv6ext->next_hdr == _ODP_IPPROTO_ROUTE) &&
+ *offset < seg_len);
+
+ if (*offset >= prs->l3_offset +
+ odp_be_to_cpu_16(ipv6->payload_len)) {
+ prs->flags.ip_err = 1;
+ return 0;
+ }
+
+ if (ipv6ext->next_hdr == _ODP_IPPROTO_FRAG)
+ prs->input_flags.ipfrag = 1;
+
+ return ipv6ext->next_hdr;
+ }
+
+ if (odp_unlikely(ipv6->next_hdr == _ODP_IPPROTO_FRAG)) {
+ prs->input_flags.ipopt = 1;
+ prs->input_flags.ipfrag = 1;
+ }
+
+ return ipv6->next_hdr;
+}
+
+/**
+ * Parser helper function for TCP
+ *
+ * Requires PARSE_TCP_BYTES bytes of contiguous packet data.
+ */
+static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr,
+ uint16_t tcp_len,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum)
+{
+ const _odp_tcphdr_t *tcp = (const _odp_tcphdr_t *)*parseptr;
+ uint32_t len = tcp->hl * 4;
+
+ if (odp_unlikely(tcp->hl < sizeof(_odp_tcphdr_t) / sizeof(uint32_t)))
+ prs->flags.tcp_err = 1;
+
+ if (chksums.chksum.tcp &&
+ !prs->input_flags.ipfrag) {
+ *l4_part_sum += odp_cpu_to_be_16(tcp_len);
+#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
+ *l4_part_sum += _ODP_IPPROTO_TCP;
+#else
+ *l4_part_sum += _ODP_IPPROTO_TCP << 8;
+#endif
+ }
+
+ *parseptr += len;
+}
+
+/**
+ * Parser helper function for UDP
+ *
+ * Requires PARSE_UDP_BYTES bytes of contiguous packet data.
+ */
+static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum)
+{
+ const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr;
+ uint32_t udplen = odp_be_to_cpu_16(udp->length);
+ uint16_t ipsec_port = odp_cpu_to_be_16(_ODP_UDP_IPSEC_PORT);
+
+ if (odp_unlikely(udplen < sizeof(_odp_udphdr_t))) {
+ prs->flags.udp_err = 1;
+ return;
+ }
+
+ if (chksums.chksum.udp &&
+ !prs->input_flags.ipfrag) {
+ if (udp->chksum == 0) {
+ prs->input_flags.l4_chksum_done = 1;
+ prs->flags.l4_chksum_err =
+ (prs->input_flags.ipv4 != 1);
+ } else {
+ *l4_part_sum += udp->length;
+#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
+ *l4_part_sum += _ODP_IPPROTO_UDP;
+#else
+ *l4_part_sum += _ODP_IPPROTO_UDP << 8;
+#endif
+ }
+ prs->input_flags.udp_chksum_zero = (udp->chksum == 0);
+ }
+
+ if (odp_unlikely(ipsec_port == udp->dst_port && udplen > 4)) {
+ uint32_t val;
+
+ memcpy(&val, udp + 1, 4);
+ if (val != 0) {
+ prs->input_flags.ipsec = 1;
+ prs->input_flags.ipsec_udp = 1;
+ }
+ }
+
+ *parseptr += sizeof(_odp_udphdr_t);
+}
+
+/**
+ * Parser helper function for SCTP
+ *
+ * Requires PARSE_SCTP_BYTES bytes of contiguous packet data.
+ */
+static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr,
+ uint16_t sctp_len,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum)
+{
+ if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) {
+ prs->flags.sctp_err = 1;
+ return;
+ }
+
+ if (chksums.chksum.sctp &&
+ !prs->input_flags.ipfrag) {
+ const _odp_sctphdr_t *sctp =
+ (const _odp_sctphdr_t *)*parseptr;
+ uint32_t crc = ~0;
+ uint32_t zero = 0;
+
+ crc = odp_hash_crc32c(sctp, sizeof(*sctp) - 4, crc);
+ crc = odp_hash_crc32c(&zero, 4, crc);
+ *l4_part_sum = crc;
+ }
+
+ *parseptr += sizeof(_odp_sctphdr_t);
+}
+
+/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */
+int _odp_packet_parse_common_l3_l4(packet_parser_t *prs,
+ const uint8_t *parseptr, uint32_t offset,
+ uint32_t frame_len, uint32_t seg_len,
+ int layer, uint16_t ethtype,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum,
+ odp_pktin_config_opt_t opt)
+{
+ uint8_t ip_proto;
+
+ prs->l3_offset = offset;
+
+ if (odp_unlikely(layer <= ODP_PROTO_LAYER_L2))
+ return prs->flags.all.error != 0;
+
+ /* Set l3 flag only for known ethtypes */
+ prs->input_flags.l3 = 1;
+
+ /* Parse Layer 3 headers */
+ switch (ethtype) {
+ case _ODP_ETHTYPE_IPV4:
+ prs->input_flags.ipv4 = 1;
+ ip_proto = parse_ipv4(prs, &parseptr, &offset, frame_len,
+ chksums, l4_part_sum);
+ prs->l4_offset = offset;
+ if (prs->flags.ip_err && opt.bit.drop_ipv4_err)
+ return -1; /* drop */
+ break;
+
+ case _ODP_ETHTYPE_IPV6:
+ prs->input_flags.ipv6 = 1;
+ ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len,
+ seg_len, chksums, l4_part_sum);
+ prs->l4_offset = offset;
+ if (prs->flags.ip_err && opt.bit.drop_ipv6_err)
+ return -1; /* drop */
+ break;
+
+ case _ODP_ETHTYPE_ARP:
+ prs->input_flags.arp = 1;
+ ip_proto = 255; /* Reserved invalid by IANA */
+ break;
+
+ default:
+ prs->input_flags.l3 = 0;
+ ip_proto = 255; /* Reserved invalid by IANA */
+ }
+
+ if (layer == ODP_PROTO_LAYER_L3)
+ return prs->flags.all.error != 0;
+
+ /* Set l4 flag only for known ip_proto */
+ prs->input_flags.l4 = 1;
+
+ /* Parse Layer 4 headers */
+ switch (ip_proto) {
+ case _ODP_IPPROTO_ICMPV4:
+ /* Fall through */
+
+ case _ODP_IPPROTO_ICMPV6:
+ prs->input_flags.icmp = 1;
+ break;
+
+ case _ODP_IPPROTO_IPIP:
+ /* Do nothing */
+ break;
+
+ case _ODP_IPPROTO_TCP:
+ if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len))
+ return -1;
+ prs->input_flags.tcp = 1;
+ parse_tcp(prs, &parseptr, frame_len - prs->l4_offset, chksums,
+ l4_part_sum);
+ if (prs->flags.tcp_err && opt.bit.drop_tcp_err)
+ return -1; /* drop */
+ break;
+
+ case _ODP_IPPROTO_UDP:
+ if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len))
+ return -1;
+ prs->input_flags.udp = 1;
+ parse_udp(prs, &parseptr, chksums, l4_part_sum);
+ if (prs->flags.udp_err && opt.bit.drop_udp_err)
+ return -1; /* drop */
+ break;
+
+ case _ODP_IPPROTO_AH:
+ prs->input_flags.ipsec = 1;
+ prs->input_flags.ipsec_ah = 1;
+ break;
+
+ case _ODP_IPPROTO_ESP:
+ prs->input_flags.ipsec = 1;
+ prs->input_flags.ipsec_esp = 1;
+ break;
+
+ case _ODP_IPPROTO_SCTP:
+ prs->input_flags.sctp = 1;
+ parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, chksums,
+ l4_part_sum);
+ if (prs->flags.sctp_err && opt.bit.drop_sctp_err)
+ return -1; /* drop */
+ break;
+
+ case _ODP_IPPROTO_NO_NEXT:
+ prs->input_flags.no_next_hdr = 1;
+ break;
+
+ default:
+ prs->input_flags.l4 = 0;
+ break;
+ }
+
+ return prs->flags.all.error != 0;
+}
diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c
index cca2d55f0..defdeb4fb 100644
--- a/platform/linux-generic/odp_pool.c
+++ b/platform/linux-generic/odp_pool.c
@@ -494,12 +494,7 @@ static void init_buffers(pool_t *pool)
for (i = 0; i < pool->num + skipped_blocks ; i++) {
int skip = 0;
-
- addr = &pool->base_addr[(i * pool->block_size) +
- pool->block_offset];
- event_hdr = addr;
- buf_hdr = addr;
- pkt_hdr = addr;
+ addr = &pool->base_addr[i * pool->block_size];
/* Skip packet buffers which cross huge page boundaries. Some
* NICs cannot handle buffers which cross page boundaries. */
@@ -517,6 +512,12 @@ static void init_buffers(pool_t *pool)
skip = 1;
}
}
+
+ addr = (uint8_t *)addr + pool->block_offset;
+ event_hdr = addr;
+ buf_hdr = addr;
+ pkt_hdr = addr;
+
if (pool->uarea_size)
uarea = &pool->uarea_base_addr[(i - skipped_blocks) *
pool->uarea_size];
diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c
index f5c3830e8..58959df31 100644
--- a/platform/linux-generic/odp_queue_lf.c
+++ b/platform/linux-generic/odp_queue_lf.c
@@ -13,6 +13,7 @@
#include <odp_debug_internal.h>
#include <odp_event_internal.h>
#include <odp_queue_basic_internal.h>
+#include <odp_types_internal.h>
#include <string.h>
#include <stdio.h>
@@ -24,9 +25,7 @@
#ifdef __SIZEOF_INT128__
-__extension__ typedef unsigned __int128 u128_t;
-
-static inline void lockfree_zero_u128(u128_t *atomic)
+static inline void lockfree_zero_u128(_odp_u128_t *atomic)
{
__atomic_store_n(atomic, 0, __ATOMIC_RELAXED);
}
@@ -40,21 +39,21 @@ static inline void lockfree_zero_u128(u128_t *atomic)
* So, these are never actually used. */
typedef struct ODP_ALIGNED(16) {
uint64_t u64[2];
-} u128_t;
+} _odp_u128_t;
-static inline u128_t lockfree_load_u128(u128_t *atomic)
+static inline _odp_u128_t lockfree_load_u128(_odp_u128_t *atomic)
{
return *atomic;
}
-static inline void lockfree_zero_u128(u128_t *atomic)
+static inline void lockfree_zero_u128(_odp_u128_t *atomic)
{
atomic->u64[0] = 0;
atomic->u64[1] = 0;
}
-static inline int lockfree_cas_acq_rel_u128(u128_t *atomic, u128_t old_val,
- u128_t new_val)
+static inline int lockfree_cas_acq_rel_u128(_odp_u128_t *atomic, _odp_u128_t old_val,
+ _odp_u128_t new_val)
{
if (atomic->u64[0] == old_val.u64[0] &&
atomic->u64[1] == old_val.u64[1]) {
@@ -75,7 +74,7 @@ static inline int lockfree_check_u128(void)
/* Node in lock-free ring */
typedef union {
- u128_t u128;
+ _odp_u128_t u128;
struct {
/* Data with lowest counter value is the head. Empty node has
diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c
index 5343c2834..7482d776d 100644
--- a/platform/linux-generic/odp_schedule_scalable.c
+++ b/platform/linux-generic/odp_schedule_scalable.c
@@ -817,23 +817,19 @@ events_dequeued:
}
/* Ingress queue empty => poll pktio RX queue */
- odp_event_t rx_evts[QUEUE_MULTI_MAX];
- int num_rx = _odp_sched_cb_pktin_poll_one(elem->pktio_idx,
- elem->rx_queue,
- rx_evts);
+ _odp_event_hdr_t *rx_evts[QUEUE_MULTI_MAX];
+ int num_rx = _odp_sched_cb_pktin_poll(elem->pktio_idx, elem->rx_queue,
+ rx_evts, QUEUE_MULTI_MAX);
+
if (odp_likely(num_rx > 0)) {
num = num_rx < num_evts ? num_rx : num_evts;
for (i = 0; i < num; i++) {
/* Return events directly to caller */
- ev[i] = rx_evts[i];
+ ev[i] = _odp_event_from_hdr(rx_evts[i]);
}
if (num_rx > num) {
/* Events remain, enqueue them */
- _odp_event_hdr_t *events[QUEUE_MULTI_MAX];
-
- for (i = num; i < num_rx; i++)
- events[i] = _odp_event_hdr(rx_evts[i]);
- i = _odp_queue_enq_sp(elem, &events[num], num_rx - num);
+ i = _odp_queue_enq_sp(elem, &rx_evts[num], num_rx - num);
/* Enqueue must succeed as the queue was empty */
ODP_ASSERT(i == num_rx - num);
}
diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c
index f5aba1a88..558c3f4fe 100644
--- a/platform/linux-generic/odp_schedule_sp.c
+++ b/platform/linux-generic/odp_schedule_sp.c
@@ -13,6 +13,7 @@
#pragma GCC diagnostic ignored "-Wzero-length-bounds"
#endif
+#include <odp/api/packet.h>
#include <odp/api/ticketlock.h>
#include <odp/api/thread.h>
#include <odp/api/plat/thread_inlines.h>
diff --git a/platform/linux-generic/odp_time.c b/platform/linux-generic/odp_time.c
index d908f02cb..b591d712a 100644
--- a/platform/linux-generic/odp_time.c
+++ b/platform/linux-generic/odp_time.c
@@ -145,15 +145,6 @@ static inline void time_wait_until(odp_time_t time)
} while (odp_time_cmp(time, cur) > 0);
}
-uint64_t odp_time_diff_ns(odp_time_t t2, odp_time_t t1)
-{
- odp_time_t time;
-
- time.u64 = t2.u64 - t1.u64;
-
- return odp_time_to_ns(time);
-}
-
odp_time_t odp_time_local_from_ns(uint64_t ns)
{
return time_from_ns(ns);
diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c
index d1d4a5bc8..1c54ab740 100644
--- a/platform/linux-generic/odp_timer.c
+++ b/platform/linux-generic/odp_timer.c
@@ -13,49 +13,52 @@
*/
#include <odp_posix_extensions.h>
-#include <errno.h>
-#include <stdlib.h>
-#include <time.h>
-#include <signal.h>
-#include <pthread.h>
-#include <unistd.h>
-#include <sys/syscall.h>
-#include <inttypes.h>
-#include <string.h>
-
-#include <odp/api/std.h>
#include <odp/api/align.h>
-#include <odp_align_internal.h>
#include <odp/api/atomic.h>
-#include <odp/api/plat/atomic_inlines.h>
-#include <odp_atomic_internal.h>
-#include <odp/api/buffer.h>
#include <odp/api/cpu.h>
-#include <odp/api/pool.h>
-#include <odp_pool_internal.h>
#include <odp/api/debug.h>
-#include <odp_debug_internal.h>
#include <odp/api/event.h>
#include <odp/api/hints.h>
-#include <odp_init_internal.h>
-#include <odp_errno_define.h>
+#include <odp/api/pool.h>
#include <odp/api/queue.h>
#include <odp/api/shared_memory.h>
#include <odp/api/spinlock.h>
-#include <odp/api/std_types.h>
+#include <odp/api/std.h>
#include <odp/api/sync.h>
#include <odp/api/time.h>
-#include <odp/api/plat/time_inlines.h>
#include <odp/api/timer.h>
+
+/* Inlined API functions */
+#include <odp/api/plat/atomic_inlines.h>
+#include <odp/api/plat/event_inlines.h>
+#include <odp/api/plat/queue_inlines.h>
+#include <odp/api/plat/time_inlines.h>
+#include <odp/api/plat/timer_inlines.h>
+
+#include <odp/api/plat/timer_inline_types.h>
+
+#include <odp_align_internal.h>
+#include <odp_atomic_internal.h>
+#include <odp_debug_internal.h>
+#include <odp_errno_define.h>
+#include <odp_event_internal.h>
+#include <odp_global_data.h>
+#include <odp_init_internal.h>
#include <odp_libconfig_internal.h>
+#include <odp_pool_internal.h>
#include <odp_queue_if.h>
#include <odp_timer_internal.h>
-#include <odp/api/plat/queue_inlines.h>
-#include <odp_global_data.h>
-#include <odp_event_internal.h>
+#include <odp_types_internal.h>
-/* Inlined API functions */
-#include <odp/api/plat/event_inlines.h>
+#include <errno.h>
+#include <inttypes.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/syscall.h>
+#include <time.h>
+#include <unistd.h>
/* One divided by one nanosecond in Hz */
#define GIGA_HZ 1000000000
@@ -94,6 +97,18 @@
#define IDX2LOCK(idx) (&timer_global->locks[(idx) % NUM_LOCKS])
#endif
+#include <odp/visibility_begin.h>
+
+/* Fill in timeout header field offsets for inline functions */
+const _odp_timeout_inline_offset_t
+_odp_timeout_inline_offset ODP_ALIGNED_CACHE = {
+ .expiration = offsetof(odp_timeout_hdr_t, expiration),
+ .timer = offsetof(odp_timeout_hdr_t, timer),
+ .user_ptr = offsetof(odp_timeout_hdr_t, user_ptr)
+};
+
+#include <odp/visibility_end.h>
+
typedef struct
#ifdef ODP_ATOMIC_U128
ODP_ALIGNED(16) /* 16-byte atomic operations need properly aligned addresses */
@@ -689,7 +704,7 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event,
/* Atomic CAS will fail if we experienced torn reads,
* retry update sequence until CAS succeeds */
} while (!_odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb,
- (_uint128_t *)&old, (_uint128_t *)&new,
+ (_odp_u128_t *)&old, (_odp_u128_t *)&new,
_ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX));
#elif __GCC_ATOMIC_LLONG_LOCK_FREE >= 2 && \
defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8
@@ -760,8 +775,8 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event,
/* We are releasing the new timeout event to some other
* thread */
_odp_atomic_u128_xchg_mm((_odp_atomic_u128_t *)tb,
- (_uint128_t *)&new,
- (_uint128_t *)&old,
+ (_odp_u128_t *)&new,
+ (_odp_u128_t *)&old,
_ODP_MEMMODEL_ACQ_RLS);
old_event = old.tmo_event;
#else
@@ -804,7 +819,7 @@ static odp_event_t timer_set_unused(timer_pool_t *tp, uint32_t idx)
new.tmo_event = ODP_EVENT_INVALID;
_odp_atomic_u128_xchg_mm((_odp_atomic_u128_t *)tb,
- (_uint128_t *)&new, (_uint128_t *)&old,
+ (_odp_u128_t *)&new, (_odp_u128_t *)&old,
_ODP_MEMMODEL_RLX);
old_event = old.tmo_event;
#else
@@ -858,8 +873,8 @@ static odp_event_t timer_cancel(timer_pool_t *tp, uint32_t idx)
/* Atomic CAS will fail if we experienced torn reads,
* retry update sequence until CAS succeeds */
} while (!_odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb,
- (_uint128_t *)&old,
- (_uint128_t *)&new,
+ (_odp_u128_t *)&old,
+ (_odp_u128_t *)&new,
_ODP_MEMMODEL_RLS,
_ODP_MEMMODEL_RLX));
old_event = old.tmo_event;
@@ -916,7 +931,7 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick)
new.tmo_event = ODP_EVENT_INVALID;
int succ = _odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb,
- (_uint128_t *)&old, (_uint128_t *)&new,
+ (_odp_u128_t *)&old, (_odp_u128_t *)&new,
_ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX);
if (succ)
tmo_event = old.tmo_event;
@@ -1468,22 +1483,6 @@ void odp_timer_pool_destroy(odp_timer_pool_t tpid)
odp_timer_pool_del(timer_pool_from_hdl(tpid));
}
-uint64_t odp_timer_tick_to_ns(odp_timer_pool_t tpid, uint64_t ticks)
-{
- (void)tpid;
-
- /* Timer ticks in API are nsec */
- return ticks;
-}
-
-uint64_t odp_timer_ns_to_tick(odp_timer_pool_t tpid, uint64_t ns)
-{
- (void)tpid;
-
- /* Timer ticks in API are nsec */
- return ns;
-}
-
uint64_t odp_timer_current_tick(odp_timer_pool_t tpid)
{
timer_pool_t *tp = timer_pool_from_hdl(tpid);
@@ -1812,19 +1811,6 @@ uint64_t odp_timer_to_u64(odp_timer_t hdl)
return _odp_pri(hdl);
}
-odp_timeout_t odp_timeout_from_event(odp_event_t ev)
-{
- /* This check not mandated by the API specification */
- if (odp_event_type(ev) != ODP_EVENT_TIMEOUT)
- ODP_ABORT("Event not a timeout");
- return (odp_timeout_t)ev;
-}
-
-odp_event_t odp_timeout_to_event(odp_timeout_t tmo)
-{
- return (odp_event_t)tmo;
-}
-
uint64_t odp_timeout_to_u64(odp_timeout_t tmo)
{
return _odp_pri(tmo);
@@ -1847,21 +1833,6 @@ int odp_timeout_fresh(odp_timeout_t tmo)
return hdr->expiration == (exp_tck & ~TMO_INACTIVE);
}
-odp_timer_t odp_timeout_timer(odp_timeout_t tmo)
-{
- return timeout_hdr(tmo)->timer;
-}
-
-uint64_t odp_timeout_tick(odp_timeout_t tmo)
-{
- return timeout_hdr(tmo)->expiration;
-}
-
-void *odp_timeout_user_ptr(odp_timeout_t tmo)
-{
- return (void *)(uintptr_t)timeout_hdr(tmo)->user_ptr;
-}
-
odp_timeout_t odp_timeout_alloc(odp_pool_t pool_hdl)
{
odp_event_t event;
diff --git a/platform/linux-generic/odp_timer_api.c b/platform/linux-generic/odp_timer_api.c
new file mode 100644
index 000000000..cd657956b
--- /dev/null
+++ b/platform/linux-generic/odp_timer_api.c
@@ -0,0 +1,11 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/timer.h>
+
+/* Non-inlined functions for ABI compat mode */
+#define _ODP_NO_INLINE
+#include <odp/api/plat/timer_inlines.h>
diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c
index fbf33e117..f02297ec9 100644
--- a/platform/linux-generic/odp_traffic_mngr.c
+++ b/platform/linux-generic/odp_traffic_mngr.c
@@ -55,45 +55,60 @@ _odp_int_name_kind_t PROFILE_TO_HANDLE_KIND[ODP_TM_NUM_PROFILES] = {
static const pkt_desc_t EMPTY_PKT_DESC = { .word = 0 };
#define MAX_PRIORITIES ODP_TM_MAX_PRIORITIES
-#define NUM_SHAPER_COLORS ODP_NUM_SHAPER_COLORS
/* Shaper BW limits in bits/sec */
#define TM_MIN_SHAPER_BW 8000ULL
#define TM_MAX_SHAPER_BW (100ULL * 1000ULL * 1000ULL * 1000ULL)
+/* Possible values for running the shaper algorithm. TM_SHAPER_GREEN means that
+ * the traffic is within the commit specification (rate and burst size),
+ * TM_SHAPER_YELLOW means that the traffic is within the peak specification
+ * (rate and burst size) and TM_SHAPER_RED means that the traffic is exceeding
+ * both its commit and peak specifications. Note that packets can also have an
+ * assigned packet color of ODP_PACKET_GREEN, ODP_PACKET_YELLOW or
+ * ODP_PACKET_RED, which has a different meaning and purpose than the shaper
+ * colors.
+ */
+typedef enum {
+ TM_SHAPER_GREEN, TM_SHAPER_YELLOW, TM_SHAPER_RED
+} tm_shaper_color_t;
+
+/* Number of enumeration values defined in tm_shaper_color_t type. */
+#define NUM_SHAPER_COLORS 3
+
static const tm_prop_t basic_prop_tbl[MAX_PRIORITIES][NUM_SHAPER_COLORS] = {
[0] = {
- [ODP_TM_SHAPER_GREEN] = { 0, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 0, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 0, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 0, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 0, DECR_BOTH },
+ [TM_SHAPER_RED] = { 0, DELAY_PKT } },
[1] = {
- [ODP_TM_SHAPER_GREEN] = { 1, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 1, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 1, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 1, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 1, DECR_BOTH },
+ [TM_SHAPER_RED] = { 1, DELAY_PKT } },
[2] = {
- [ODP_TM_SHAPER_GREEN] = { 2, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 2, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 2, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 2, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 2, DECR_BOTH },
+ [TM_SHAPER_RED] = { 2, DELAY_PKT } },
[3] = {
- [ODP_TM_SHAPER_GREEN] = { 3, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 3, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 3, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 3, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 3, DECR_BOTH },
+ [TM_SHAPER_RED] = { 3, DELAY_PKT } },
[4] = {
- [ODP_TM_SHAPER_GREEN] = { 4, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 4, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 4, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 4, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 4, DECR_BOTH },
+ [TM_SHAPER_RED] = { 4, DELAY_PKT } },
[5] = {
- [ODP_TM_SHAPER_GREEN] = { 5, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 5, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 5, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 5, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 5, DECR_BOTH },
+ [TM_SHAPER_RED] = { 5, DELAY_PKT } },
[6] = {
- [ODP_TM_SHAPER_GREEN] = { 6, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 6, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 6, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 6, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 6, DECR_BOTH },
+ [TM_SHAPER_RED] = { 6, DELAY_PKT } },
[7] = {
- [ODP_TM_SHAPER_GREEN] = { 7, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 7, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 7, DELAY_PKT } }
+ [TM_SHAPER_GREEN] = { 7, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 7, DECR_BOTH },
+ [TM_SHAPER_RED] = { 7, DELAY_PKT } }
};
#define MAX_SHAPER_PROFILES 128
@@ -1076,30 +1091,30 @@ static odp_bool_t run_shaper(tm_system_t *tm_system,
pkt_desc_t *pkt_desc,
uint8_t priority)
{
- odp_tm_shaper_color_t shaper_color;
+ tm_shaper_color_t shaper_color;
tm_shaper_params_t *shaper_params;
odp_bool_t output_change;
tm_prop_t propagation;
shaper_params = shaper_obj->shaper_params;
- shaper_color = ODP_TM_SHAPER_GREEN;
+ shaper_color = TM_SHAPER_GREEN;
if (shaper_params) {
update_shaper_elapsed_time(tm_system, shaper_params,
shaper_obj);
if (shaper_params->enabled) {
if (0 < shaper_obj->commit_cnt)
- shaper_color = ODP_TM_SHAPER_GREEN;
+ shaper_color = TM_SHAPER_GREEN;
else if (!shaper_params->dual_rate)
- shaper_color = ODP_TM_SHAPER_RED;
+ shaper_color = TM_SHAPER_RED;
else if (shaper_obj->peak_cnt <= 0)
- shaper_color = ODP_TM_SHAPER_RED;
+ shaper_color = TM_SHAPER_RED;
else
- shaper_color = ODP_TM_SHAPER_YELLOW;
+ shaper_color = TM_SHAPER_YELLOW;
- if (shaper_color == ODP_TM_SHAPER_GREEN)
+ if (shaper_color == TM_SHAPER_GREEN)
tm_system->shaper_green_cnt++;
- else if (shaper_color == ODP_TM_SHAPER_YELLOW)
+ else if (shaper_color == TM_SHAPER_YELLOW)
tm_system->shaper_yellow_cnt++;
else
tm_system->shaper_red_cnt++;
@@ -1994,6 +2009,12 @@ static void tm_queue_cnts_decrement(tm_system_t *tm_system,
odp_atomic_sub_u64(&queue_cnts->byte_cnt, frame_len);
}
+static inline void activate_packet_aging(odp_packet_hdr_t *pkt_hdr)
+{
+ if (odp_unlikely(pkt_hdr->p.flags.tx_aging))
+ pkt_hdr->tx_aging_ns = pkt_hdr->tx_aging_ns + odp_time_global_ns();
+}
+
static int tm_enqueue(tm_system_t *tm_system,
tm_queue_obj_t *tm_queue_obj,
odp_packet_t pkt)
@@ -2028,6 +2049,7 @@ static int tm_enqueue(tm_system_t *tm_system,
if (tm_queue_obj->ordered_enqueue)
_odp_sched_fn->order_lock();
+ activate_packet_aging(packet_hdr(pkt));
rc = input_work_queue_append(tm_system, &work_item);
if (tm_queue_obj->ordered_enqueue)
@@ -2224,6 +2246,11 @@ static void tm_egress_marking(tm_system_t *tm_system, odp_packet_t odp_pkt)
}
}
+static inline odp_bool_t is_packet_aged(odp_packet_hdr_t *pkt_hdr)
+{
+ return pkt_hdr->p.flags.tx_aging && pkt_hdr->tx_aging_ns < odp_time_global_ns();
+}
+
static void tm_send_pkt(tm_system_t *tm_system, uint32_t max_sends)
{
tm_queue_obj_t *tm_queue_obj;
@@ -2250,9 +2277,13 @@ static void tm_send_pkt(tm_system_t *tm_system, uint32_t max_sends)
tm_system->egress_pkt_desc = EMPTY_PKT_DESC;
if (tm_system->egress.egress_kind == ODP_TM_EGRESS_PKT_IO) {
- ret = odp_pktout_send(tm_system->pktout, &odp_pkt, 1);
+ pktio_entry = get_pktio_entry(tm_system->pktout.pktio);
+ if (odp_unlikely(_odp_pktio_tx_aging_enabled(pktio_entry) &&
+ is_packet_aged(packet_hdr(odp_pkt))))
+ ret = 0; /* Aged packet handled as a discard */
+ else
+ ret = odp_pktout_send(tm_system->pktout, &odp_pkt, 1);
if (odp_unlikely(ret != 1)) {
- pktio_entry = get_pktio_entry(tm_system->pktout.pktio);
if (odp_unlikely(_odp_pktio_tx_compl_enabled(pktio_entry)))
_odp_pktio_allocate_and_send_tx_compl_events(pktio_entry,
&odp_pkt, 1);
@@ -2581,7 +2612,11 @@ static int tm_capabilities(odp_tm_capabilities_t capabilities[],
cap_ptr->vlan_marking_supported = true;
cap_ptr->ecn_marking_supported = true;
cap_ptr->drop_prec_marking_supported = true;
- cap_ptr->tm_queue_threshold = true;
+
+ cap_ptr->tm_queue_threshold.byte = true;
+ cap_ptr->tm_queue_threshold.packet = true;
+ cap_ptr->tm_queue_threshold.byte_and_packet = true;
+
cap_ptr->tm_queue_query_flags = (ODP_TM_QUERY_PKT_CNT |
ODP_TM_QUERY_BYTE_CNT |
ODP_TM_QUERY_THRESHOLDS);
@@ -2617,7 +2652,10 @@ static int tm_capabilities(odp_tm_capabilities_t capabilities[],
per_level_cap->tm_node_dual_slope_supported = true;
per_level_cap->fair_queuing_supported = true;
per_level_cap->weights_supported = true;
- per_level_cap->tm_node_threshold = true;
+
+ per_level_cap->tm_node_threshold.byte = true;
+ per_level_cap->tm_node_threshold.packet = true;
+ per_level_cap->tm_node_threshold.byte_and_packet = true;
}
cap_ptr->queue_stats.counter.discards = 1;
@@ -2691,7 +2729,11 @@ static void tm_system_capabilities_set(odp_tm_capabilities_t *cap_ptr,
cap_ptr->ecn_marking_supported = req_ptr->ecn_marking_needed;
cap_ptr->drop_prec_marking_supported =
req_ptr->drop_prec_marking_needed;
- cap_ptr->tm_queue_threshold = threshold;
+
+ cap_ptr->tm_queue_threshold.byte = threshold;
+ cap_ptr->tm_queue_threshold.packet = threshold;
+ cap_ptr->tm_queue_threshold.byte_and_packet = threshold;
+
cap_ptr->tm_queue_query_flags = (ODP_TM_QUERY_PKT_CNT |
ODP_TM_QUERY_BYTE_CNT |
ODP_TM_QUERY_THRESHOLDS);
@@ -2743,7 +2785,10 @@ static void tm_system_capabilities_set(odp_tm_capabilities_t *cap_ptr,
per_level_cap->tm_node_dual_slope_supported = dual_slope;
per_level_cap->fair_queuing_supported = true;
per_level_cap->weights_supported = true;
- per_level_cap->tm_node_threshold = threshold;
+
+ per_level_cap->tm_node_threshold.byte = threshold;
+ per_level_cap->tm_node_threshold.packet = threshold;
+ per_level_cap->tm_node_threshold.byte_and_packet = threshold;
}
cap_ptr->queue_stats.counter.discards = 1;
diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c
index 815ec61b9..030560b0d 100644
--- a/platform/linux-generic/pktio/dpdk.c
+++ b/platform/linux-generic/pktio/dpdk.c
@@ -57,12 +57,14 @@
#include <rte_string_fns.h>
#include <rte_version.h>
-#if RTE_VERSION < RTE_VERSION_NUM(19, 8, 0, 0)
- #define rte_ether_addr ether_addr
- #define rte_ipv4_hdr ipv4_hdr
- #define rte_ipv6_hdr ipv6_hdr
- #define rte_tcp_hdr tcp_hdr
- #define rte_udp_hdr udp_hdr
+#if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0)
+ #define RTE_MBUF_F_RX_RSS_HASH PKT_RX_RSS_HASH
+ #define RTE_MBUF_F_TX_IPV4 PKT_TX_IPV4
+ #define RTE_MBUF_F_TX_IPV6 PKT_TX_IPV6
+ #define RTE_MBUF_F_TX_IP_CKSUM PKT_TX_IP_CKSUM
+ #define RTE_MBUF_F_TX_UDP_CKSUM PKT_TX_UDP_CKSUM
+ #define RTE_MBUF_F_TX_TCP_CKSUM PKT_TX_TCP_CKSUM
+ #define RTE_MEMPOOL_REGISTER_OPS MEMPOOL_REGISTER_OPS
#endif
/* NUMA is not supported on all platforms */
@@ -72,17 +74,7 @@
#define numa_num_configured_nodes() 1
#endif
-#if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 0)
-#define rte_log_set_global_level rte_set_log_level
-#endif
-
-/* Release notes v19.11: "Changed the mempool allocation behaviour
- * so that objects no longer cross pages by default" */
-#if RTE_VERSION >= RTE_VERSION_NUM(19, 11, 0, 0)
-#define MEMPOOL_FLAGS MEMPOOL_F_NO_IOVA_CONTIG
-#else
#define MEMPOOL_FLAGS 0
-#endif
#if _ODP_DPDK_ZERO_COPY
ODP_STATIC_ASSERT(CONFIG_PACKET_HEADROOM == RTE_PKTMBUF_HEADROOM,
@@ -107,11 +99,7 @@ ODP_STATIC_ASSERT((DPDK_NB_MBUF % DPDK_MEMPOOL_CACHE_SIZE == 0) &&
#define DPDK_MIN_RX_BURST 4
/* Limits for setting link MTU */
-#if RTE_VERSION >= RTE_VERSION_NUM(19, 11, 0, 0)
#define DPDK_MTU_MIN (RTE_ETHER_MIN_MTU + _ODP_ETHHDR_LEN)
-#else
-#define DPDK_MTU_MIN (68 + _ODP_ETHHDR_LEN)
-#endif
#define DPDK_MTU_MAX (9000 + _ODP_ETHHDR_LEN)
/** DPDK runtime configuration options */
@@ -334,12 +322,16 @@ static void pktmbuf_init(struct rte_mempool *mp, void *opaque_arg ODP_UNUSED,
static struct rte_mempool *mbuf_pool_create(const char *name,
pool_t *pool_entry)
{
+ odp_shm_info_t shm_info;
struct rte_mempool *mp = NULL;
struct rte_pktmbuf_pool_private mbp_priv;
struct rte_mempool_objsz sz;
unsigned int elt_size = pool_entry->dpdk_elt_size;
- unsigned int num = pool_entry->num;
+ unsigned int num = pool_entry->num, populated = 0;
uint32_t total_size;
+ uint64_t page_size, offset = 0, remainder = 0;
+ uint8_t *addr;
+ int ret;
if (!(pool_entry->mem_from_huge_pages)) {
ODP_ERR("DPDK requires memory is allocated from huge pages\n");
@@ -348,11 +340,17 @@ static struct rte_mempool *mbuf_pool_create(const char *name,
if (pool_entry->seg_len < RTE_MBUF_DEFAULT_BUF_SIZE) {
ODP_ERR("Some NICs need at least %dB buffers to not segment "
- "standard ethernet frames. Increase pool seg_len.\n",
- RTE_MBUF_DEFAULT_BUF_SIZE);
+ "standard ethernet frames. Increase pool seg_len.\n",
+ RTE_MBUF_DEFAULT_BUF_SIZE);
goto fail;
}
+ if (odp_shm_info(pool_entry->shm, &shm_info)) {
+ ODP_ERR("Failed to query SHM info.\n");
+ goto fail;
+ }
+
+ page_size = shm_info.page_size;
total_size = rte_mempool_calc_obj_size(elt_size, MEMPOOL_FLAGS, &sz);
if (total_size != pool_entry->block_size) {
ODP_ERR("DPDK pool block size not matching to ODP pool: "
@@ -361,10 +359,7 @@ static struct rte_mempool *mbuf_pool_create(const char *name,
goto fail;
}
- /* Skipped buffers have to be taken into account to populate pool
- * properly. */
- mp = rte_mempool_create_empty(name, num + pool_entry->skipped_blocks,
- elt_size, cache_size(num),
+ mp = rte_mempool_create_empty(name, num, elt_size, cache_size(num),
sizeof(struct rte_pktmbuf_pool_private),
rte_socket_id(), MEMPOOL_FLAGS);
if (mp == NULL) {
@@ -380,16 +375,34 @@ static struct rte_mempool *mbuf_pool_create(const char *name,
}
mbp_priv.mbuf_data_room_size = pool_entry->headroom +
- pool_entry->seg_len;
+ pool_entry->seg_len + pool_entry->tailroom;
mbp_priv.mbuf_priv_size = RTE_ALIGN(sizeof(odp_packet_hdr_t),
RTE_MBUF_PRIV_ALIGN);
rte_pktmbuf_pool_init(mp, &mbp_priv);
- num = rte_mempool_populate_iova(mp, (char *)pool_entry->base_addr,
- RTE_BAD_IOVA, pool_entry->shm_size,
- NULL, NULL);
- if (num <= 0) {
- ODP_ERR("Failed to populate mempool: %d\n", num);
+ /* DPDK expects buffers that would be crossing a hugepage boundary to be aligned to the
+ * boundary. This isn't the case with ODP pools as boundary-crossing buffers are skipped
+ * and unused but still part of the pool. Thus, populate the mempool with several virtually
+ * and physically contiguous chunks as dictated by the skipped buffers. */
+ for (uint64_t i = 0; i < pool_entry->shm_size; i += page_size) {
+ remainder = (page_size - offset) % total_size;
+ addr = pool_entry->base_addr + i + offset;
+ ret = rte_mempool_populate_iova(mp, (char *)addr, rte_mem_virt2iova(addr),
+ page_size - remainder - offset,
+ NULL, NULL);
+
+ if (ret <= 0) {
+ ODP_ERR("Failed to populate mempool: %d\n", ret);
+ goto fail;
+ }
+
+ populated += ret;
+ offset = remainder ? total_size - remainder : 0;
+ }
+
+ if (populated != num) {
+ ODP_ERR("Failed to populate mempool with all requested blocks, populated: %u, "
+ "requested: %u\n", populated, num);
goto fail;
}
@@ -556,7 +569,7 @@ static struct rte_mempool_ops odp_pool_ops = {
.get_count = pool_get_count
};
-MEMPOOL_REGISTER_OPS(odp_pool_ops)
+RTE_MEMPOOL_REGISTER_OPS(odp_pool_ops)
static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry,
odp_packet_t pkt_table[],
@@ -574,9 +587,9 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry,
pkt_dpdk_t *pkt_dpdk = pkt_priv(pktio_entry);
odp_pool_t pool = pkt_dpdk->pool;
odp_pktin_config_opt_t pktin_cfg = pktio_entry->s.config.pktin;
- odp_proto_layer_t parse_layer = pktio_entry->s.config.parser.layer;
odp_pktio_t input = pktio_entry->s.handle;
uint16_t frame_offset = pktio_entry->s.pktin_frame_offset;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
/* Allocate maximum sized packets */
max_len = pkt_dpdk->data_room;
@@ -591,8 +604,6 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry,
}
for (i = 0; i < num; i++) {
- odp_packet_hdr_t parsed_hdr;
-
mbuf = mbuf_table[i];
if (odp_unlikely(mbuf->nb_segs != 1)) {
ODP_ERR("Segmented buffers not supported\n");
@@ -603,31 +614,33 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry,
odp_prefetch(data);
pkt_len = rte_pktmbuf_pkt_len(mbuf);
+ pkt = pkt_table[i];
+ pkt_hdr = packet_hdr(pkt);
- if (pktio_cls_enabled(pktio_entry)) {
+ if (layer) {
uint32_t supported_ptypes = pkt_dpdk->supported_ptypes;
- packet_parse_reset(&parsed_hdr, 1);
- packet_set_len(&parsed_hdr, pkt_len);
- if (_odp_dpdk_packet_parse_common(&parsed_hdr.p, data,
- pkt_len, pkt_len,
- mbuf,
- ODP_PROTO_LAYER_ALL,
- supported_ptypes,
+ packet_parse_reset(pkt_hdr, 1);
+ if (_odp_dpdk_packet_parse_common(&pkt_hdr->p, data,
+ pkt_len, pkt_len, mbuf,
+ layer, supported_ptypes,
pktin_cfg)) {
odp_packet_free(pkt_table[i]);
rte_pktmbuf_free(mbuf);
continue;
}
- if (_odp_cls_classify_packet(pktio_entry,
- (const uint8_t *)data,
- pkt_len, pkt_len, &pool,
- &parsed_hdr, false))
- goto fail;
+
+ if (pktio_cls_enabled(pktio_entry)) {
+ if (_odp_cls_classify_packet(pktio_entry,
+ (const uint8_t *)data,
+ &pool, pkt_hdr)) {
+ odp_packet_free(pkt_table[i]);
+ rte_pktmbuf_free(mbuf);
+ continue;
+ }
+ }
}
- pkt = pkt_table[i];
- pkt_hdr = packet_hdr(pkt);
pull_tail(pkt_hdr, max_len - pkt_len);
if (frame_offset)
pull_head(pkt_hdr, frame_offset);
@@ -637,22 +650,7 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry,
pkt_hdr->input = input;
- if (pktio_cls_enabled(pktio_entry)) {
- _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr);
- } else if (parse_layer != ODP_PROTO_LAYER_NONE) {
- uint32_t supported_ptypes = pkt_dpdk->supported_ptypes;
-
- if (_odp_dpdk_packet_parse_layer(pkt_hdr, mbuf,
- parse_layer,
- supported_ptypes,
- pktin_cfg)) {
- odp_packet_free(pkt);
- rte_pktmbuf_free(mbuf);
- continue;
- }
- }
-
- if (mbuf->ol_flags & PKT_RX_RSS_HASH)
+ if (mbuf->ol_flags & RTE_MBUF_F_RX_RSS_HASH)
packet_set_flow_hash(pkt_hdr, mbuf->hash.rss);
packet_set_ts(pkt_hdr, ts);
@@ -753,12 +751,12 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg,
mbuf->l2_len = pkt_p->l3_offset - pkt_p->l2_offset;
if (l3_proto_v4)
- mbuf->ol_flags = PKT_TX_IPV4;
+ mbuf->ol_flags = RTE_MBUF_F_TX_IPV4;
else
- mbuf->ol_flags = PKT_TX_IPV6;
+ mbuf->ol_flags = RTE_MBUF_F_TX_IPV6;
if (ipv4_chksum_pkt) {
- mbuf->ol_flags |= PKT_TX_IP_CKSUM;
+ mbuf->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM;
((struct rte_ipv4_hdr *)l3_hdr)->hdr_checksum = 0;
mbuf->l3_len = _ODP_IPV4HDR_IHL(*(uint8_t *)l3_hdr) * 4;
@@ -772,12 +770,12 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg,
l4_hdr = (void *)(mbuf_data + pkt_p->l4_offset);
if (udp_chksum_pkt) {
- mbuf->ol_flags |= PKT_TX_UDP_CKSUM;
+ mbuf->ol_flags |= RTE_MBUF_F_TX_UDP_CKSUM;
((struct rte_udp_hdr *)l4_hdr)->dgram_cksum =
phdr_csum(l3_proto_v4, l3_hdr, mbuf->ol_flags);
} else if (tcp_chksum_pkt) {
- mbuf->ol_flags |= PKT_TX_TCP_CKSUM;
+ mbuf->ol_flags |= RTE_MBUF_F_TX_TCP_CKSUM;
((struct rte_tcp_hdr *)l4_hdr)->cksum =
phdr_csum(l3_proto_v4, l3_hdr, mbuf->ol_flags);
@@ -863,9 +861,9 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry,
int i, nb_pkts;
odp_pool_t pool;
odp_pktin_config_opt_t pktin_cfg;
- odp_proto_layer_t parse_layer;
odp_pktio_t input;
pkt_dpdk_t *pkt_dpdk;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
prefetch_pkt(mbuf_table[0]);
@@ -874,15 +872,12 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry,
pool = pkt_dpdk->pool;
set_flow_hash = pkt_dpdk->opt.set_flow_hash;
pktin_cfg = pktio_entry->s.config.pktin;
- parse_layer = pktio_entry->s.config.parser.layer;
input = pktio_entry->s.handle;
if (odp_likely(mbuf_num > 1))
prefetch_pkt(mbuf_table[1]);
for (i = 0; i < mbuf_num; i++) {
- odp_packet_hdr_t parsed_hdr;
-
if (odp_likely((i + 2) < mbuf_num))
prefetch_pkt(mbuf_table[i + 2]);
@@ -895,30 +890,27 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry,
data = rte_pktmbuf_mtod(mbuf, char *);
pkt_len = rte_pktmbuf_pkt_len(mbuf);
-
pkt_hdr = pkt_hdr_from_mbuf(mbuf);
+ packet_init(pkt_hdr, pkt_len);
- if (pktio_cls_enabled(pktio_entry)) {
+ if (layer) {
uint32_t supported_ptypes = pkt_dpdk->supported_ptypes;
- packet_parse_reset(&parsed_hdr, 1);
- packet_set_len(&parsed_hdr, pkt_len);
- if (_odp_dpdk_packet_parse_common(&parsed_hdr.p, data,
- pkt_len, pkt_len,
- mbuf,
- ODP_PROTO_LAYER_ALL,
- supported_ptypes,
+ if (_odp_dpdk_packet_parse_common(&pkt_hdr->p, data,
+ pkt_len, pkt_len, mbuf,
+ layer, supported_ptypes,
pktin_cfg)) {
rte_pktmbuf_free(mbuf);
continue;
}
- if (_odp_cls_classify_packet(pktio_entry,
- (const uint8_t *)data,
- pkt_len, pkt_len, &pool,
- &parsed_hdr, false)) {
- ODP_ERR("Unable to classify packet\n");
- rte_pktmbuf_free(mbuf);
- continue;
+
+ if (pktio_cls_enabled(pktio_entry)) {
+ if (_odp_cls_classify_packet(pktio_entry,
+ (const uint8_t *)data,
+ &pool, pkt_hdr)) {
+ rte_pktmbuf_free(mbuf);
+ continue;
+ }
}
}
@@ -926,23 +918,9 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry,
* are supported. */
pkt_hdr->seg_data = data;
- packet_init(pkt_hdr, pkt_len);
pkt_hdr->input = input;
- if (pktio_cls_enabled(pktio_entry)) {
- _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr);
- } else if (parse_layer != ODP_PROTO_LAYER_NONE) {
- uint32_t supported_ptypes = pkt_dpdk->supported_ptypes;
-
- if (_odp_dpdk_packet_parse_layer(pkt_hdr, mbuf,
- parse_layer,
- supported_ptypes,
- pktin_cfg)) {
- rte_pktmbuf_free(mbuf);
- continue;
- }
- }
- if (set_flow_hash && (mbuf->ol_flags & PKT_RX_RSS_HASH))
+ if (set_flow_hash && (mbuf->ol_flags & RTE_MBUF_F_RX_RSS_HASH))
packet_set_flow_hash(pkt_hdr, mbuf->hash.rss);
packet_set_ts(pkt_hdr, ts);
@@ -1227,11 +1205,6 @@ static int dpdk_close(pktio_entry_t *pktio_entry)
rte_pktmbuf_free(pkt_dpdk->rx_cache[i].s.pkt[idx++]);
}
-#if RTE_VERSION < RTE_VERSION_NUM(17, 8, 0, 0)
- if (pktio_entry->s.state != PKTIO_STATE_OPENED)
- rte_eth_dev_close(pkt_dpdk->port_id);
-#endif
-
return 0;
}
@@ -1381,18 +1354,6 @@ static void dpdk_mempool_free(struct rte_mempool *mp, void *arg ODP_UNUSED)
rte_mempool_free(mp);
}
-/* RTE_ETH_FOREACH_DEV was introduced in v17.8, but causes a build error in
- * v18.2 (only a warning, but our build system treats warnings as errors). */
-#if (RTE_VERSION >= RTE_VERSION_NUM(18, 2, 0, 0)) && \
- (RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0))
- #define ETH_FOREACH_DEV(p) \
- for (p = rte_eth_find_next(0); \
- (unsigned int)p < (unsigned int)RTE_MAX_ETHPORTS; \
- p = rte_eth_find_next(p + 1))
-#elif RTE_VERSION >= RTE_VERSION_NUM(17, 8, 0, 0)
- #define ETH_FOREACH_DEV(p) RTE_ETH_FOREACH_DEV(p)
-#endif
-
static int dpdk_pktio_term(void)
{
uint16_t port_id;
@@ -1400,11 +1361,9 @@ static int dpdk_pktio_term(void)
if (!odp_global_rw->dpdk_initialized)
return 0;
-#if RTE_VERSION >= RTE_VERSION_NUM(17, 8, 0, 0)
- ETH_FOREACH_DEV(port_id) {
+ RTE_ETH_FOREACH_DEV(port_id) {
rte_eth_dev_close(port_id);
}
-#endif
if (!_ODP_DPDK_ZERO_COPY)
rte_mempool_walk(dpdk_mempool_free, NULL);
@@ -1574,16 +1533,18 @@ static int dpdk_init_capability(pktio_entry_t *pktio_entry,
/* Check if setting MTU is supported */
ret = rte_eth_dev_set_mtu(pkt_dpdk->port_id, pkt_dpdk->mtu - _ODP_ETHHDR_LEN);
- if (ret == 0) {
+ /* From DPDK 21.11 onwards, calling rte_eth_dev_set_mtu() before device is configured with
+ * rte_eth_dev_configure() will result in failure. The least hacky (unfortunately still
+ * very hacky) way to continue checking the support is to take into account that the
+ * function will fail earlier with -ENOTSUP if MTU setting is not supported by device than
+ * if the device was not yet configured. */
+ if (ret != -ENOTSUP) {
capa->set_op.op.maxlen = 1;
capa->maxlen.equal = true;
capa->maxlen.min_input = DPDK_MTU_MIN;
capa->maxlen.max_input = pkt_dpdk->mtu_max;
capa->maxlen.min_output = DPDK_MTU_MIN;
capa->maxlen.max_output = pkt_dpdk->mtu_max;
- } else if (ret != -ENOTSUP) {
- ODP_ERR("Failed to set interface MTU: %d\n", ret);
- return -1;
}
ptype_cnt = rte_eth_dev_get_supported_ptypes(pkt_dpdk->port_id,
@@ -1669,15 +1630,6 @@ static int dpdk_init_capability(pktio_entry_t *pktio_entry,
* mode change. Use system call for them. */
static void promisc_mode_check(pkt_dpdk_t *pkt_dpdk)
{
-#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0)
- /* Enable and disable calls do not have return value */
- rte_eth_promiscuous_enable(pkt_dpdk->port_id);
-
- if (!rte_eth_promiscuous_get(pkt_dpdk->port_id))
- pkt_dpdk->vdev_sysc_promisc = 1;
-
- rte_eth_promiscuous_disable(pkt_dpdk->port_id);
-#else
int ret;
ret = rte_eth_promiscuous_enable(pkt_dpdk->port_id);
@@ -1689,7 +1641,6 @@ static void promisc_mode_check(pkt_dpdk_t *pkt_dpdk)
if (ret)
pkt_dpdk->vdev_sysc_promisc = 1;
-#endif
}
static int dpdk_open(odp_pktio_t id ODP_UNUSED,
@@ -1735,18 +1686,17 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED,
pkt_dpdk->pool = pool;
- /* rte_eth_dev_count() was removed in v18.05 */
-#if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0)
- if (rte_eth_dev_count() == 0) {
-#else
if (rte_eth_dev_count_avail() == 0) {
-#endif
ODP_ERR("No DPDK ports found\n");
return -1;
}
memset(&dev_info, 0, sizeof(struct rte_eth_dev_info));
- rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info);
+ ret = rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info);
+ if (ret) {
+ ODP_ERR("Failed to read device info: %d\n", ret);
+ return -1;
+ }
/* Initialize runtime options */
if (init_options(pktio_entry, &dev_info)) {
@@ -1765,18 +1715,10 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED,
promisc_mode_check(pkt_dpdk);
-#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0)
- ret = 0;
- if (pkt_dpdk->opt.multicast_en)
- rte_eth_allmulticast_enable(pkt_dpdk->port_id);
- else
- rte_eth_allmulticast_disable(pkt_dpdk->port_id);
-#else
if (pkt_dpdk->opt.multicast_en)
ret = rte_eth_allmulticast_enable(pkt_dpdk->port_id);
else
ret = rte_eth_allmulticast_disable(pkt_dpdk->port_id);
-#endif
/* Not supported by all PMDs, so ignore the return value */
if (ret)
diff --git a/platform/linux-generic/pktio/dpdk_parse.c b/platform/linux-generic/pktio/dpdk_parse.c
index 0984f06c5..f28f18921 100644
--- a/platform/linux-generic/pktio/dpdk_parse.c
+++ b/platform/linux-generic/pktio/dpdk_parse.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2018, Linaro Limited
- * Copyright (c) 2021, Nokia
+ * Copyright (c) 2021-2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -26,9 +26,19 @@
#if defined(__PPC64__) && defined(vector)
#undef vector
#endif
+#include <rte_version.h>
+
+#if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0)
+ #define RTE_MBUF_F_RX_IP_CKSUM_MASK PKT_RX_IP_CKSUM_MASK
+ #define RTE_MBUF_F_RX_IP_CKSUM_GOOD PKT_RX_IP_CKSUM_GOOD
+ #define RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN PKT_RX_IP_CKSUM_UNKNOWN
+ #define RTE_MBUF_F_RX_L4_CKSUM_MASK PKT_RX_L4_CKSUM_MASK
+ #define RTE_MBUF_F_RX_L4_CKSUM_GOOD PKT_RX_L4_CKSUM_GOOD
+ #define RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN PKT_RX_L4_CKSUM_UNKNOWN
+#endif
-#define IP4_CSUM_RESULT(ol_flags) (ol_flags & PKT_RX_IP_CKSUM_MASK)
-#define L4_CSUM_RESULT(ol_flags) (ol_flags & PKT_RX_L4_CKSUM_MASK)
+#define IP4_CSUM_RESULT(ol_flags) ((ol_flags) & RTE_MBUF_F_RX_IP_CKSUM_MASK)
+#define L4_CSUM_RESULT(ol_flags) ((ol_flags) & RTE_MBUF_F_RX_L4_CKSUM_MASK)
/** Parser helper function for Ethernet packets */
static inline uint16_t dpdk_parse_eth(packet_parser_t *prs,
@@ -174,9 +184,9 @@ static inline uint8_t dpdk_parse_ipv4(packet_parser_t *prs,
if (do_csum) {
uint64_t packet_csum_result = IP4_CSUM_RESULT(mbuf_ol);
- if (packet_csum_result == PKT_RX_IP_CKSUM_GOOD) {
+ if (packet_csum_result == RTE_MBUF_F_RX_IP_CKSUM_GOOD) {
prs->input_flags.l3_chksum_done = 1;
- } else if (packet_csum_result != PKT_RX_IP_CKSUM_UNKNOWN) {
+ } else if (packet_csum_result != RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN) {
prs->input_flags.l3_chksum_done = 1;
prs->flags.ip_err = 1;
prs->flags.l3_chksum_err = 1;
@@ -302,9 +312,9 @@ static inline void dpdk_parse_tcp(packet_parser_t *prs,
if (do_csum) {
uint64_t packet_csum_result = L4_CSUM_RESULT(mbuf_ol);
- if (packet_csum_result == PKT_RX_L4_CKSUM_GOOD) {
+ if (packet_csum_result == RTE_MBUF_F_RX_L4_CKSUM_GOOD) {
prs->input_flags.l4_chksum_done = 1;
- } else if (packet_csum_result != PKT_RX_L4_CKSUM_UNKNOWN) {
+ } else if (packet_csum_result != RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN) {
prs->input_flags.l4_chksum_done = 1;
prs->flags.tcp_err = 1;
prs->flags.l4_chksum_err = 1;
@@ -332,9 +342,9 @@ static inline void dpdk_parse_udp(packet_parser_t *prs,
if (do_csum) {
uint64_t packet_csum_result = L4_CSUM_RESULT(mbuf_ol);
- if (packet_csum_result == PKT_RX_L4_CKSUM_GOOD) {
+ if (packet_csum_result == RTE_MBUF_F_RX_L4_CKSUM_GOOD) {
prs->input_flags.l4_chksum_done = 1;
- } else if (packet_csum_result != PKT_RX_L4_CKSUM_UNKNOWN) {
+ } else if (packet_csum_result != RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN) {
if (prs->input_flags.ipv4 && !udp->chksum) {
prs->input_flags.l4_chksum_done = 1;
} else {
diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c
index b3232cded..3e21efecd 100644
--- a/platform/linux-generic/pktio/loop.c
+++ b/platform/linux-generic/pktio/loop.c
@@ -18,6 +18,7 @@
#include <odp/api/plat/packet_flag_inlines.h>
#include <odp/api/plat/queue_inlines.h>
+#include <odp_parse_internal.h>
#include <odp_classification_internal.h>
#include <odp_debug_internal.h>
#include <odp_errno_define.h>
@@ -26,6 +27,7 @@
#include <odp_ipsec_internal.h>
#include <odp_packet_internal.h>
#include <odp_packet_io_internal.h>
+#include <odp_macros_internal.h>
#include <odp_queue_if.h>
#include <protocols/eth.h>
@@ -158,7 +160,11 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
odp_time_t ts_val;
odp_time_t *ts = NULL;
int num_rx = 0;
- int failed = 0;
+ int packets = 0, errors = 0;
+ uint32_t octets = 0;
+ const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
+ const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin;
if (odp_unlikely(num > QUEUE_MULTI_MAX))
num = QUEUE_MULTI_MAX;
@@ -168,8 +174,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
queue = pkt_priv(pktio_entry)->loopq;
nbr = odp_queue_deq_multi(queue, (odp_event_t *)hdr_tbl, num);
- if (pktio_entry->s.config.pktin.bit.ts_all ||
- pktio_entry->s.config.pktin.bit.ts_ptp) {
+ if (opt.bit.ts_all || opt.bit.ts_ptp) {
ts_val = odp_time_global();
ts = &ts_val;
}
@@ -181,61 +186,64 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
pkt_len = odp_packet_len(pkt);
pkt_hdr = packet_hdr(pkt);
- packet_parse_reset(pkt_hdr, 1);
- if (pktio_cls_enabled(pktio_entry)) {
- odp_packet_t new_pkt;
- odp_pool_t new_pool;
+ if (layer) {
uint8_t *pkt_addr;
- uint8_t buf[PACKET_PARSE_SEG_LEN];
+ uint8_t buf[PARSE_BYTES];
int ret;
uint32_t seg_len = odp_packet_seg_len(pkt);
+ uint64_t l4_part_sum = 0;
/* Make sure there is enough data for the packet
* parser in the case of a segmented packet. */
- if (odp_unlikely(seg_len < PACKET_PARSE_SEG_LEN &&
- pkt_len > PACKET_PARSE_SEG_LEN)) {
- odp_packet_copy_to_mem(pkt, 0,
- PACKET_PARSE_SEG_LEN,
- buf);
- seg_len = PACKET_PARSE_SEG_LEN;
+ if (odp_unlikely(seg_len < PARSE_BYTES &&
+ pkt_len > seg_len)) {
+ seg_len = MIN(pkt_len, PARSE_BYTES);
+ odp_packet_copy_to_mem(pkt, 0, seg_len, buf);
pkt_addr = buf;
} else {
pkt_addr = odp_packet_data(pkt);
}
- ret = _odp_cls_classify_packet(pktio_entry, pkt_addr,
- pkt_len, seg_len,
- &new_pool, pkt_hdr, true);
- if (ret) {
- failed++;
+ packet_parse_reset(pkt_hdr, 1);
+ ret = _odp_packet_parse_common(&pkt_hdr->p, pkt_addr, pkt_len,
+ seg_len, layer, chksums,
+ &l4_part_sum, opt);
+ if (ret)
+ errors++;
+
+ if (ret < 0) {
odp_packet_free(pkt);
continue;
}
- if (new_pool != odp_packet_pool(pkt)) {
- new_pkt = odp_packet_copy(pkt, new_pool);
-
- odp_packet_free(pkt);
+ if (pktio_cls_enabled(pktio_entry)) {
+ odp_packet_t new_pkt;
+ odp_pool_t new_pool;
- if (new_pkt == ODP_PACKET_INVALID) {
- failed++;
+ ret = _odp_cls_classify_packet(pktio_entry, pkt_addr,
+ &new_pool, pkt_hdr);
+ if (ret) {
+ odp_packet_free(pkt);
continue;
}
- pkt = new_pkt;
- pkt_hdr = packet_hdr(new_pkt);
+ if (new_pool != odp_packet_pool(pkt)) {
+ new_pkt = odp_packet_copy(pkt, new_pool);
+
+ odp_packet_free(pkt);
+
+ if (new_pkt == ODP_PACKET_INVALID) {
+ pktio_entry->s.stats.in_discards++;
+ continue;
+ }
+
+ pkt = new_pkt;
+ pkt_hdr = packet_hdr(new_pkt);
+ }
}
- } else {
- odp_packet_parse_param_t param;
-
- /*
- * Use odp_packet_parse() which can handle segmented
- * packets.
- */
- param.proto = ODP_PROTO_ETH;
- param.last_layer = pktio_entry->s.config.parser.layer;
- param.chksums = pktio_entry->s.in_chksums;
- odp_packet_parse(packet_handle(pkt_hdr), 0, &param);
+
+ if (layer >= ODP_PROTO_LAYER_L4)
+ _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
}
packet_set_ts(pkt_hdr, ts);
@@ -247,12 +255,17 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
odp_packet_has_ipsec(pkt))
_odp_ipsec_try_inline(&pkt);
- pktio_entry->s.stats.in_octets += pkt_len;
+ if (!pkt_hdr->p.flags.all.error) {
+ octets += pkt_len;
+ packets++;
+ }
+
pkts[num_rx++] = pkt;
}
- pktio_entry->s.stats.in_errors += failed;
- pktio_entry->s.stats.in_packets += num_rx - failed;
+ pktio_entry->s.stats.in_octets += octets;
+ pktio_entry->s.stats.in_packets += packets;
+ pktio_entry->s.stats.in_errors += errors;
odp_ticketlock_unlock(&pktio_entry->s.rxl);
@@ -533,6 +546,7 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry)
capa->stats.pktio.counter.in_octets = 1;
capa->stats.pktio.counter.in_packets = 1;
capa->stats.pktio.counter.in_errors = 1;
+ capa->stats.pktio.counter.in_discards = 1;
capa->stats.pktio.counter.out_octets = 1;
capa->stats.pktio.counter.out_packets = 1;
capa->stats.pktin_queue.counter.octets = 1;
diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c
index 0b1345e50..94b88e21e 100644
--- a/platform/linux-generic/pktio/netmap.c
+++ b/platform/linux-generic/pktio/netmap.c
@@ -16,6 +16,7 @@
#include <odp/api/time.h>
#include <odp/api/plat/time_inlines.h>
+#include <odp_parse_internal.h>
#include <odp_packet_io_internal.h>
#include <odp_packet_io_stats.h>
#include <odp_ethtool_stats.h>
@@ -822,11 +823,14 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry,
odp_packet_t pkt;
odp_pool_t pool = pkt_priv(pktio_entry)->pool;
odp_packet_hdr_t *pkt_hdr;
- odp_packet_hdr_t parsed_hdr;
int i;
int num;
uint32_t max_len;
uint16_t frame_offset = pktio_entry->s.pktin_frame_offset;
+ int num_rx = 0;
+ const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
+ const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin;
/* Allocate maximum sized packets */
max_len = pkt_priv(pktio_entry)->mtu;
@@ -837,45 +841,50 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry,
for (i = 0; i < num; i++) {
netmap_slot_t slot;
uint16_t len;
+ const uint8_t *buf;
+ uint64_t l4_part_sum = 0;
slot = slot_tbl[i];
len = slot.len;
+ buf = (const uint8_t *)slot.buf;
odp_prefetch(slot.buf);
- if (pktio_cls_enabled(pktio_entry)) {
- if (_odp_cls_classify_packet(pktio_entry,
- (const uint8_t *)slot.buf, len,
- len, &pool, &parsed_hdr, true))
- goto fail;
+ pkt = pkt_tbl[num_rx];
+ pkt_hdr = packet_hdr(pkt);
+
+ if (layer) {
+ if (_odp_packet_parse_common(&pkt_hdr->p, buf, len, len,
+ layer, chksums, &l4_part_sum, opt) < 0)
+ continue;
+
+ if (pktio_cls_enabled(pktio_entry)) {
+ if (_odp_cls_classify_packet(pktio_entry, buf, &pool,
+ pkt_hdr))
+ continue;
+ }
}
- pkt = pkt_tbl[i];
- pkt_hdr = packet_hdr(pkt);
pull_tail(pkt_hdr, max_len - len);
if (frame_offset)
pull_head(pkt_hdr, frame_offset);
if (odp_packet_copy_from_mem(pkt, 0, len, slot.buf) != 0)
- goto fail;
+ break;
pkt_hdr->input = pktio_entry->s.handle;
- if (pktio_cls_enabled(pktio_entry))
- _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr);
- else
- _odp_packet_parse_layer(pkt_hdr,
- pktio_entry->s.config.parser.layer,
- pktio_entry->s.in_chksums);
+ if (layer >= ODP_PROTO_LAYER_L4)
+ _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
packet_set_ts(pkt_hdr, ts);
+ num_rx++;
}
- return i;
+ if (num_rx < num)
+ odp_packet_free_multi(&pkt_tbl[num_rx], num - num_rx);
-fail:
- odp_packet_free_multi(&pkt_tbl[i], num - i);
- return i;
+ return num_rx;
}
static inline int netmap_recv_desc(pktio_entry_t *pktio_entry,
@@ -1042,11 +1051,11 @@ static int netmap_recv_tmo(pktio_entry_t *pktio_entry, int index,
}
static int netmap_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[],
- int num_q, odp_packet_t pkt_table[], int num,
- unsigned *from, uint64_t usecs)
+ uint32_t num_q, odp_packet_t pkt_table[], int num,
+ uint32_t *from, uint64_t usecs)
{
struct timeval timeout;
- int i;
+ uint32_t i;
int ret;
int maxfd = -1, maxfd2;
fd_set readfds;
diff --git a/platform/linux-generic/pktio/null.c b/platform/linux-generic/pktio/null.c
index 90e113ec6..a6498da8c 100644
--- a/platform/linux-generic/pktio/null.c
+++ b/platform/linux-generic/pktio/null.c
@@ -71,9 +71,9 @@ static int null_recv_tmo(pktio_entry_t *pktio_entry ODP_UNUSED,
}
static int null_recv_mq_tmo(pktio_entry_t *pktio_entry[] ODP_UNUSED,
- int index[] ODP_UNUSED, int num_q ODP_UNUSED,
+ int index[] ODP_UNUSED, uint32_t num_q ODP_UNUSED,
odp_packet_t pkt_table[] ODP_UNUSED,
- int num ODP_UNUSED, unsigned *from ODP_UNUSED,
+ int num ODP_UNUSED, uint32_t *from ODP_UNUSED,
uint64_t usecs)
{
struct timeval timeout;
diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c
index 7290626c2..af94ffa72 100644
--- a/platform/linux-generic/pktio/pcap.c
+++ b/platform/linux-generic/pktio/pcap.c
@@ -46,6 +46,7 @@
#include <odp/api/plat/packet_inlines.h>
+#include <odp_parse_internal.h>
#include <odp_classification_internal.h>
#include <odp_debug_internal.h>
#include <odp_global_data.h>
@@ -248,7 +249,12 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
pkt_pcap_t *pcap = pkt_priv(pktio_entry);
odp_time_t ts_val;
odp_time_t *ts = NULL;
+ int packets = 0, errors = 0;
+ uint32_t octets = 0;
uint16_t frame_offset = pktio_entry->s.pktin_frame_offset;
+ const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
+ const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin;
odp_ticketlock_lock(&pktio_entry->s.rxl);
@@ -256,8 +262,7 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
odp_ticketlock_unlock(&pktio_entry->s.rxl);
return 0;
}
- if (pktio_entry->s.config.pktin.bit.ts_all ||
- pktio_entry->s.config.pktin.bit.ts_ptp)
+ if (opt.bit.ts_all || opt.bit.ts_ptp)
ts = &ts_val;
for (i = 0; i < num; ) {
@@ -291,42 +296,64 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
break;
}
- if (pktio_cls_enabled(pktio_entry)) {
- odp_packet_t new_pkt;
+ if (layer) {
+ uint64_t l4_part_sum = 0;
- ret = _odp_cls_classify_packet(pktio_entry, data,
- pkt_len, pkt_len,
- &new_pool, pkt_hdr, true);
- if (ret) {
+ ret = _odp_packet_parse_common(&pkt_hdr->p, data, pkt_len,
+ pkt_len, layer, chksums,
+ &l4_part_sum, opt);
+ if (ret)
+ errors++;
+
+ if (ret < 0) {
odp_packet_free(pkt);
continue;
}
- if (new_pool != pcap->pool) {
- new_pkt = odp_packet_copy(pkt, new_pool);
- odp_packet_free(pkt);
+ if (pktio_cls_enabled(pktio_entry)) {
+ odp_packet_t new_pkt;
- if (odp_unlikely(new_pkt == ODP_PACKET_INVALID))
+ ret = _odp_cls_classify_packet(pktio_entry, data,
+ &new_pool, pkt_hdr);
+ if (ret) {
+ odp_packet_free(pkt);
continue;
+ }
+ if (new_pool != pcap->pool) {
+ new_pkt = odp_packet_copy(pkt, new_pool);
+
+ odp_packet_free(pkt);
+
+ if (odp_unlikely(new_pkt == ODP_PACKET_INVALID)) {
+ pktio_entry->s.stats.in_discards++;
+ continue;
+ }
- pkt = new_pkt;
- pkt_hdr = packet_hdr(new_pkt);
+ pkt = new_pkt;
+ pkt_hdr = packet_hdr(new_pkt);
+ }
}
- } else {
- _odp_packet_parse_layer(pkt_hdr,
- pktio_entry->s.config.parser.layer,
- pktio_entry->s.in_chksums);
+
+ if (layer >= ODP_PROTO_LAYER_L4)
+ _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
}
- pktio_entry->s.stats.in_octets += pkt_hdr->frame_len;
packet_set_ts(pkt_hdr, ts);
pkt_hdr->input = pktio_entry->s.handle;
+ if (!pkt_hdr->p.flags.all.error) {
+ octets += pkt_len;
+ packets++;
+ }
+
pkts[i] = pkt;
i++;
}
- pktio_entry->s.stats.in_packets += i;
+
+ pktio_entry->s.stats.in_octets += octets;
+ pktio_entry->s.stats.in_packets += packets;
+ pktio_entry->s.stats.in_errors += errors;
odp_ticketlock_unlock(&pktio_entry->s.rxl);
@@ -441,6 +468,8 @@ static int pcapif_capability(pktio_entry_t *pktio_entry ODP_UNUSED,
capa->stats.pktio.counter.in_octets = 1;
capa->stats.pktio.counter.in_packets = 1;
+ capa->stats.pktio.counter.in_discards = 1;
+ capa->stats.pktio.counter.in_errors = 1;
capa->stats.pktio.counter.out_octets = 1;
capa->stats.pktio.counter.out_packets = 1;
diff --git a/platform/linux-generic/pktio/pktio_common.c b/platform/linux-generic/pktio/pktio_common.c
index d2f4d7219..1f3cf951e 100644
--- a/platform/linux-generic/pktio/pktio_common.c
+++ b/platform/linux-generic/pktio/pktio_common.c
@@ -11,13 +11,13 @@
static int sock_recv_mq_tmo_select(pktio_entry_t * const *entry,
const int index[],
- unsigned int num_q, unsigned int *from,
+ uint32_t num_q, uint32_t *from,
odp_packet_t packets[], int num,
uint64_t usecs, fd_set *readfds,
int maxfd)
{
struct timeval timeout;
- unsigned int i;
+ uint32_t i;
int ret;
for (i = 0; i < num_q; i++) {
@@ -50,17 +50,17 @@ static int sock_recv_mq_tmo_select(pktio_entry_t * const *entry,
}
int _odp_sock_recv_mq_tmo_try_int_driven(const struct odp_pktin_queue_t queues[],
- unsigned int num_q, unsigned int *from,
+ uint32_t num_q, uint32_t *from,
odp_packet_t packets[], int num,
uint64_t usecs, int *trial_successful)
{
- unsigned int i;
+ uint32_t i;
pktio_entry_t *entry[num_q];
int index[num_q];
fd_set readfds;
int maxfd = -1;
- int (*impl)(pktio_entry_t *entry[], int index[], int num_q,
- odp_packet_t packets[], int num, unsigned int *from,
+ int (*impl)(pktio_entry_t *entry[], int index[], uint32_t num_q,
+ odp_packet_t packets[], int num, uint32_t *from,
uint64_t wait_usecs) = NULL;
int impl_set = 0;
diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c
index 81c178b9d..9d1bbe545 100644
--- a/platform/linux-generic/pktio/socket.c
+++ b/platform/linux-generic/pktio/socket.c
@@ -15,12 +15,14 @@
#include <odp/api/ticketlock.h>
#include <odp_socket_common.h>
+#include <odp_parse_internal.h>
#include <odp_packet_internal.h>
#include <odp_packet_io_internal.h>
#include <odp_packet_io_stats.h>
#include <odp_debug_internal.h>
#include <odp_errno_define.h>
#include <odp_classification_internal.h>
+#include <odp_macros_internal.h>
#include <sys/socket.h>
#include <stdio.h>
@@ -233,6 +235,9 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
int i;
uint16_t frame_offset = pktio_entry->s.pktin_frame_offset;
uint32_t alloc_len = pkt_sock->mtu + frame_offset;
+ const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
+ const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin;
memset(msgvec, 0, sizeof(msgvec));
@@ -249,8 +254,7 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
recv_msgs = recvmmsg(sockfd, msgvec, nb_pkts, MSG_DONTWAIT, NULL);
odp_ticketlock_unlock(&pkt_sock->rx_lock);
- if (pktio_entry->s.config.pktin.bit.ts_all ||
- pktio_entry->s.config.pktin.bit.ts_ptp) {
+ if (opt.bit.ts_all || opt.bit.ts_ptp) {
ts_val = odp_time_global();
ts = &ts_val;
}
@@ -262,25 +266,48 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
uint16_t pkt_len = msgvec[i].msg_len;
int ret;
+ uint64_t l4_part_sum = 0;
if (odp_unlikely(msgvec[i].msg_hdr.msg_flags & MSG_TRUNC)) {
odp_packet_free(pkt);
ODP_DBG("dropped truncated packet\n");
continue;
}
- if (pktio_cls_enabled(pktio_entry)) {
- uint16_t seg_len = pkt_len;
- if (msgvec[i].msg_hdr.msg_iov->iov_len < pkt_len)
- seg_len = msgvec[i].msg_hdr.msg_iov->iov_len;
+ ret = odp_packet_trunc_tail(&pkt, odp_packet_len(pkt) - pkt_len,
+ NULL, NULL);
+ if (ret < 0) {
+ ODP_ERR("trunc_tail failed");
+ odp_packet_free(pkt);
+ continue;
+ }
+
+ if (layer) {
+ uint8_t buf[PARSE_BYTES];
+ uint16_t seg_len = msgvec[i].msg_hdr.msg_iov->iov_len;
- if (_odp_cls_classify_packet(pktio_entry, base, pkt_len,
- seg_len, &pool, pkt_hdr,
- true)) {
- ODP_ERR("_odp_cls_classify_packet failed");
+ /* Make sure there is enough data for the packet
+ * parser in the case of a segmented packet. */
+ if (odp_unlikely(seg_len < PARSE_BYTES && pkt_len > seg_len)) {
+ seg_len = MIN(pkt_len, PARSE_BYTES);
+ odp_packet_copy_to_mem(pkt, 0, seg_len, buf);
+ base = buf;
+ }
+
+ if (_odp_packet_parse_common(&pkt_hdr->p, base, pkt_len,
+ seg_len, layer, chksums,
+ &l4_part_sum, opt) < 0) {
odp_packet_free(pkt);
continue;
}
+
+ if (pktio_cls_enabled(pktio_entry)) {
+ if (_odp_cls_classify_packet(pktio_entry, base, &pool,
+ pkt_hdr)) {
+ odp_packet_free(pkt);
+ continue;
+ }
+ }
}
/* Don't receive packets sent by ourselves */
@@ -290,20 +317,10 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
continue;
}
- ret = odp_packet_trunc_tail(&pkt, odp_packet_len(pkt) - pkt_len,
- NULL, NULL);
- if (ret < 0) {
- ODP_ERR("trunk_tail failed");
- odp_packet_free(pkt);
- continue;
- }
-
pkt_hdr->input = pktio_entry->s.handle;
- if (!pktio_cls_enabled(pktio_entry))
- _odp_packet_parse_layer(pkt_hdr,
- pktio_entry->s.config.parser.layer,
- pktio_entry->s.in_chksums);
+ if (layer >= ODP_PROTO_LAYER_L4)
+ _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
packet_set_ts(pkt_hdr, ts);
@@ -359,11 +376,11 @@ static int sock_recv_tmo(pktio_entry_t *pktio_entry, int index,
}
static int sock_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[],
- int num_q, odp_packet_t pkt_table[], int num,
- unsigned *from, uint64_t usecs)
+ uint32_t num_q, odp_packet_t pkt_table[], int num,
+ uint32_t *from, uint64_t usecs)
{
struct timeval timeout;
- int i;
+ uint32_t i;
int ret;
int maxfd = -1, maxfd2;
fd_set readfds;
diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c
index f84834610..7824b0e91 100644
--- a/platform/linux-generic/pktio/socket_mmap.c
+++ b/platform/linux-generic/pktio/socket_mmap.c
@@ -16,6 +16,7 @@
#include <odp/api/plat/packet_inlines.h>
#include <odp_socket_common.h>
+#include <odp_parse_internal.h>
#include <odp_packet_internal.h>
#include <odp_packet_io_internal.h>
#include <odp_packet_io_stats.h>
@@ -150,9 +151,11 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry,
odp_pool_t pool = pkt_sock->pool;
uint16_t frame_offset = pktio_entry->s.pktin_frame_offset;
uint16_t vlan_len = 0;
+ const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
+ const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin;
- if (pktio_entry->s.config.pktin.bit.ts_all ||
- pktio_entry->s.config.pktin.bit.ts_ptp)
+ if (opt.bit.ts_all || opt.bit.ts_ptp)
ts = &ts_val;
ring = &pkt_sock->rx_ring;
@@ -163,8 +166,8 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry,
struct tpacket2_hdr *tp_hdr;
odp_packet_t pkt;
odp_packet_hdr_t *hdr;
- odp_packet_hdr_t parsed_hdr;
int ret;
+ uint64_t l4_part_sum = 0;
tp_hdr = (void *)next_ptr;
@@ -212,18 +215,29 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry,
continue;
}
- if (pktio_cls_enabled(pktio_entry)) {
- if (_odp_cls_classify_packet(pktio_entry, pkt_buf, pkt_len,
- pkt_len, &pool, &parsed_hdr,
- true)) {
+ hdr = packet_hdr(pkt);
+
+ if (layer) {
+ if (_odp_packet_parse_common(&hdr->p, pkt_buf, pkt_len,
+ pkt_len, layer, chksums,
+ &l4_part_sum, opt) < 0) {
odp_packet_free(pkt);
tp_hdr->tp_status = TP_STATUS_KERNEL;
frame_num = next_frame_num;
continue;
}
+
+ if (pktio_cls_enabled(pktio_entry)) {
+ if (_odp_cls_classify_packet(pktio_entry, pkt_buf,
+ &pool, hdr)) {
+ odp_packet_free(pkt);
+ tp_hdr->tp_status = TP_STATUS_KERNEL;
+ frame_num = next_frame_num;
+ continue;
+ }
+ }
}
- hdr = packet_hdr(pkt);
if (frame_offset)
pull_head(hdr, frame_offset);
@@ -276,12 +290,8 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry,
hdr->input = pktio_entry->s.handle;
- if (pktio_cls_enabled(pktio_entry))
- _odp_packet_copy_cls_md(hdr, &parsed_hdr);
- else
- _odp_packet_parse_layer(hdr,
- pktio_entry->s.config.parser.layer,
- pktio_entry->s.in_chksums);
+ if (layer >= ODP_PROTO_LAYER_L4)
+ _odp_packet_l4_chksum(hdr, chksums, l4_part_sum);
packet_set_ts(hdr, ts);
@@ -727,11 +737,11 @@ static int sock_mmap_recv_tmo(pktio_entry_t *pktio_entry, int index,
}
static int sock_mmap_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[],
- int num_q, odp_packet_t pkt_table[], int num,
- unsigned *from, uint64_t usecs)
+ uint32_t num_q, odp_packet_t pkt_table[], int num,
+ uint32_t *from, uint64_t usecs)
{
struct timeval timeout;
- int i;
+ uint32_t i;
int ret;
int maxfd = -1, maxfd2;
fd_set readfds;
diff --git a/platform/linux-generic/pktio/tap.c b/platform/linux-generic/pktio/tap.c
index d298c8982..40c46b43c 100644
--- a/platform/linux-generic/pktio/tap.c
+++ b/platform/linux-generic/pktio/tap.c
@@ -38,6 +38,7 @@
#include <odp/api/plat/packet_inlines.h>
+#include <odp_parse_internal.h>
#include <odp_debug_internal.h>
#include <odp_socket_common.h>
#include <odp_packet_internal.h>
@@ -286,14 +287,27 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data,
odp_packet_hdr_t *pkt_hdr;
odp_packet_hdr_t parsed_hdr;
int num;
+ uint64_t l4_part_sum = 0;
uint16_t frame_offset = pktio_entry->s.pktin_frame_offset;
-
- if (pktio_cls_enabled(pktio_entry)) {
- if (_odp_cls_classify_packet(pktio_entry, data, len, len,
- &pkt_priv(pktio_entry)->pool,
- &parsed_hdr, true)) {
+ const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
+ const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin;
+
+ if (layer) {
+ packet_parse_reset(&parsed_hdr, 1);
+ packet_set_len(&parsed_hdr, len);
+ if (_odp_packet_parse_common(&parsed_hdr.p, data, len, len, layer,
+ chksums, &l4_part_sum, opt) < 0) {
return ODP_PACKET_INVALID;
}
+
+ if (pktio_cls_enabled(pktio_entry)) {
+ if (_odp_cls_classify_packet(pktio_entry, data,
+ &pkt_priv(pktio_entry)->pool,
+ &parsed_hdr)) {
+ return ODP_PACKET_INVALID;
+ }
+ }
}
num = _odp_packet_alloc_multi(pkt_priv(pktio_entry)->pool,
@@ -312,12 +326,12 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data,
return ODP_PACKET_INVALID;
}
- if (pktio_cls_enabled(pktio_entry))
+ if (layer) {
_odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr);
- else
- _odp_packet_parse_layer(pkt_hdr,
- pktio_entry->s.config.parser.layer,
- pktio_entry->s.in_chksums);
+
+ if (layer >= ODP_PROTO_LAYER_L4)
+ _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
+ }
packet_set_ts(pkt_hdr, ts);
pkt_hdr->input = pktio_entry->s.handle;
@@ -335,6 +349,7 @@ static int tap_pktio_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
uint8_t buf[mtu];
odp_time_t ts_val;
odp_time_t *ts = NULL;
+ int num_rx = 0;
odp_ticketlock_lock(&pktio_entry->s.rxl);
@@ -355,14 +370,15 @@ static int tap_pktio_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
break;
}
- pkts[i] = pack_odp_pkt(pktio_entry, buf, retval, ts);
- if (pkts[i] == ODP_PACKET_INVALID)
+ pkts[num_rx] = pack_odp_pkt(pktio_entry, buf, retval, ts);
+ if (pkts[num_rx] == ODP_PACKET_INVALID)
break;
+ num_rx++;
}
odp_ticketlock_unlock(&pktio_entry->s.rxl);
- return i;
+ return num_rx;
}
static int tap_pktio_send_lockless(pktio_entry_t *pktio_entry,