diff options
Diffstat (limited to 'platform/linux-generic')
42 files changed, 618 insertions, 224 deletions
diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index ce8b0ba09..e762148aa 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -39,6 +39,7 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/event_inline_types.h \ include/odp/api/plat/event_vector_inline_types.h \ include/odp/api/plat/hash_inlines.h \ + include/odp/api/plat/ipsec_inlines.h \ include/odp/api/plat/packet_flag_inlines.h \ include/odp/api/plat/packet_inline_types.h \ include/odp/api/plat/packet_inlines.h \ @@ -82,6 +83,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/hash.h \ include-abi/odp/api/abi/init.h \ include-abi/odp/api/abi/ipsec.h \ + include-abi/odp/api/abi/ipsec_types.h \ include-abi/odp/api/abi/packet.h \ include-abi/odp/api/abi/packet_types.h \ include-abi/odp/api/abi/packet_flags.h \ @@ -288,6 +290,7 @@ __LIB__libodp_linux_la_SOURCES += \ odp_cpu_api.c \ odp_event_api.c \ odp_hash_api.c \ + odp_ipsec_api.c \ odp_packet_api.c \ odp_packet_flags_api.c \ odp_packet_io_api.c \ diff --git a/platform/linux-generic/include-abi/odp/api/abi/ipsec.h b/platform/linux-generic/include-abi/odp/api/abi/ipsec.h index a04bb1741..1817e5564 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/ipsec.h +++ b/platform/linux-generic/include-abi/odp/api/abi/ipsec.h @@ -1,4 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,7 +8,7 @@ /** * @file * - * ODP IPSEC API - platform specific types + * ODP IPsec */ #ifndef ODP_API_ABI_IPSEC_H_ @@ -17,16 +18,8 @@ extern "C" { #endif -#include <odp/api/std_types.h> -#include <odp/api/plat/strong_types.h> - -/** @ingroup odp_ipsec - * @{ - */ - -typedef ODP_HANDLE_T(odp_ipsec_sa_t); - -#define ODP_IPSEC_SA_INVALID _odp_cast_scalar(odp_ipsec_sa_t, 0) +/* Inlined API functions */ +#include <odp/api/plat/ipsec_inlines.h> /** * @} diff --git a/platform/linux-generic/include-abi/odp/api/abi/ipsec_types.h b/platform/linux-generic/include-abi/odp/api/abi/ipsec_types.h new file mode 100644 index 000000000..376666ded --- /dev/null +++ b/platform/linux-generic/include-abi/odp/api/abi/ipsec_types.h @@ -0,0 +1,41 @@ +/* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP IPsec platform specific types + */ + +#ifndef ODP_API_ABI_IPSEC_TYPES_H_ +#define ODP_API_ABI_IPSEC_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/std_types.h> + +#include <odp/api/plat/strong_types.h> + +/** @ingroup odp_ipsec + * @{ + */ + +typedef ODP_HANDLE_T(odp_ipsec_sa_t); + +#define ODP_IPSEC_SA_INVALID _odp_cast_scalar(odp_ipsec_sa_t, 0) + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include-abi/odp/api/abi/packet_io_types.h b/platform/linux-generic/include-abi/odp/api/abi/packet_io_types.h index 3106f26a7..1692e71b1 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/packet_io_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/packet_io_types.h @@ -47,6 +47,8 @@ typedef struct odp_pktout_queue_t { #define ODP_PKTIN_NO_WAIT 0 +#define ODP_PKTIN_MAX_QUEUES 64 + #define ODP_PKTOUT_MAX_QUEUES 64 #define ODP_PKTIO_STATS_EXTRA_NAME_LEN 64 diff --git a/platform/linux-generic/include-abi/odp/api/abi/packet_types.h b/platform/linux-generic/include-abi/odp/api/abi/packet_types.h index be2cb9df6..4da9332ba 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/packet_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/packet_types.h @@ -49,39 +49,6 @@ typedef ODP_HANDLE_T(odp_packet_tx_compl_t); #define ODP_PACKET_OFFSET_INVALID 0xffff -typedef uint8_t odp_proto_l2_type_t; - -#define ODP_PROTO_L2_TYPE_NONE 0 -#define ODP_PROTO_L2_TYPE_ETH 1 - -typedef uint8_t odp_proto_l3_type_t; - -#define ODP_PROTO_L3_TYPE_NONE 0 -#define ODP_PROTO_L3_TYPE_ARP 1 -#define ODP_PROTO_L3_TYPE_RARP 2 -#define ODP_PROTO_L3_TYPE_MPLS 3 -#define ODP_PROTO_L3_TYPE_IPV4 4 -#define ODP_PROTO_L3_TYPE_IPV6 6 - -typedef uint8_t odp_proto_l4_type_t; - -/* Numbers from IANA Assigned Internet Protocol Numbers list */ -#define ODP_PROTO_L4_TYPE_NONE 0 -#define ODP_PROTO_L4_TYPE_ICMPV4 1 -#define ODP_PROTO_L4_TYPE_IGMP 2 -#define ODP_PROTO_L4_TYPE_IPV4 4 -#define ODP_PROTO_L4_TYPE_TCP 6 -#define ODP_PROTO_L4_TYPE_UDP 17 -#define ODP_PROTO_L4_TYPE_IPV6 41 -#define ODP_PROTO_L4_TYPE_GRE 47 -#define ODP_PROTO_L4_TYPE_ESP 50 -#define ODP_PROTO_L4_TYPE_AH 51 -#define ODP_PROTO_L4_TYPE_ICMPV6 58 -#define ODP_PROTO_L4_TYPE_NO_NEXT 59 -#define ODP_PROTO_L4_TYPE_IPCOMP 108 -#define ODP_PROTO_L4_TYPE_SCTP 132 -#define ODP_PROTO_L4_TYPE_ROHC 142 - typedef enum { ODP_PACKET_GREEN = 0, ODP_PACKET_YELLOW = 1, diff --git a/platform/linux-generic/include-abi/odp/api/abi/shared_memory.h b/platform/linux-generic/include-abi/odp/api/abi/shared_memory.h index 648bd1ee3..551d49e30 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/shared_memory.h +++ b/platform/linux-generic/include-abi/odp/api/abi/shared_memory.h @@ -31,6 +31,9 @@ typedef ODP_HANDLE_T(odp_shm_t); #define ODP_SHM_NAME_LEN 32 +#define ODP_SHM_IOVA_INVALID ((uint64_t)-1) +#define ODP_SHM_PA_INVALID ODP_SHM_IOVA_INVALID + /** * @} */ diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h b/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h index 9689ddd06..f64a176f5 100644 --- a/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h @@ -26,6 +26,8 @@ typedef struct _odp_buffer_inline_offset_t { } _odp_buffer_inline_offset_t; +extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; + /** @endcond */ #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h index d876f60f4..34d4b5675 100644 --- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h @@ -19,15 +19,13 @@ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -extern const _odp_event_inline_offset_t _odp_event_inline_offset; -extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; -extern const _odp_pool_inline_offset_t _odp_pool_inline; - #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline #define odp_buffer_from_event __odp_buffer_from_event + #define odp_buffer_from_event_multi __odp_buffer_from_event_multi #define odp_buffer_to_event __odp_buffer_to_event + #define odp_buffer_to_event_multi __odp_buffer_to_event_multi #define odp_buffer_addr __odp_buffer_addr #define odp_buffer_size __odp_buffer_size #define odp_buffer_pool __odp_buffer_pool @@ -43,11 +41,23 @@ _ODP_INLINE odp_buffer_t odp_buffer_from_event(odp_event_t ev) return (odp_buffer_t)ev; } +_ODP_INLINE void odp_buffer_from_event_multi(odp_buffer_t buf[], const odp_event_t ev[], int num) +{ + for (int i = 0; i < num; i++) + buf[i] = odp_buffer_from_event(ev[i]); +} + _ODP_INLINE odp_event_t odp_buffer_to_event(odp_buffer_t buf) { return (odp_event_t)buf; } +_ODP_INLINE void odp_buffer_to_event_multi(const odp_buffer_t buf[], odp_event_t ev[], int num) +{ + for (int i = 0; i < num; i++) + ev[i] = odp_buffer_to_event(buf[i]); +} + _ODP_INLINE void *odp_buffer_addr(odp_buffer_t buf) { return _odp_event_hdr_field((odp_event_t)buf, void *, base_data); diff --git a/platform/linux-generic/include/odp/api/plat/event_inline_types.h b/platform/linux-generic/include/odp/api/plat/event_inline_types.h index 77a0deecb..caa075871 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/event_inline_types.h @@ -33,6 +33,8 @@ typedef struct _odp_event_inline_offset_t { } _odp_event_inline_offset_t; +extern const _odp_event_inline_offset_t _odp_event_inline_offset; + /** @endcond */ #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index dd8e9249c..37c015b21 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -16,9 +16,6 @@ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -extern const _odp_event_inline_offset_t _odp_event_inline_offset; -extern const _odp_packet_inline_offset_t _odp_packet_inline; - #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline diff --git a/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h b/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h index 723e1a3d1..773f5171c 100644 --- a/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h @@ -40,6 +40,8 @@ typedef struct _odp_event_vector_inline_offset_t { } _odp_event_vector_inline_offset_t; +extern const _odp_event_vector_inline_offset_t _odp_event_vector_inline; + /** @endcond */ #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp/api/plat/ipsec_inlines.h b/platform/linux-generic/include/odp/api/plat/ipsec_inlines.h new file mode 100644 index 000000000..1d1f6ec61 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/ipsec_inlines.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_IPSEC_INLINES_H_ +#define ODP_PLAT_IPSEC_INLINES_H_ + +#include <odp/api/event.h> +#include <odp/api/ipsec_types.h> +#include <odp/api/packet.h> + +#include <odp/api/plat/debug_inlines.h> +#include <odp/api/plat/packet_inline_types.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_ipsec_packet_from_event __odp_ipsec_packet_from_event + #define odp_ipsec_packet_to_event __odp_ipsec_packet_to_event + #define odp_ipsec_result __odp_ipsec_result +#else + #define _ODP_INLINE +#endif + +_ODP_INLINE odp_packet_t odp_ipsec_packet_from_event(odp_event_t ev) +{ + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); + _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_IPSEC); + + return odp_packet_from_event(ev); +} + +_ODP_INLINE odp_event_t odp_ipsec_packet_to_event(odp_packet_t pkt) +{ + return odp_packet_to_event(pkt); +} + +_ODP_INLINE int odp_ipsec_result(odp_ipsec_packet_result_t *result, odp_packet_t pkt) +{ + odp_ipsec_packet_result_t *res; + + _ODP_ASSERT(result != NULL); + _ODP_ASSERT(odp_packet_subtype(pkt) == ODP_EVENT_PACKET_IPSEC); + + res = _odp_pkt_get_ptr(pkt, odp_ipsec_packet_result_t, ipsec_ctx); + + *result = *res; + + return 0; +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h index 04ad9e968..9330d89f8 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h @@ -20,8 +20,6 @@ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -extern const _odp_packet_inline_offset_t _odp_packet_inline; - static inline uint64_t _odp_packet_input_flags(odp_packet_t pkt) { return _odp_pkt_get(pkt, uint64_t, input_flags); diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index 2ae0829c3..ae03457f9 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -52,9 +52,12 @@ typedef struct _odp_packet_inline_offset_t { uint16_t flags; uint16_t subtype; uint16_t cls_mark; + uint16_t ipsec_ctx; } _odp_packet_inline_offset_t; +extern const _odp_packet_inline_offset_t _odp_packet_inline; + /* Packet input & protocol flags */ typedef union { /* All input flags */ diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 21d184ac5..01d47d837 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -111,9 +111,6 @@ int _odp_packet_copy_from_mem_seg(odp_packet_t pkt, uint32_t offset, int _odp_packet_copy_to_mem_seg(odp_packet_t pkt, uint32_t offset, uint32_t len, void *dst); -extern const _odp_packet_inline_offset_t _odp_packet_inline; -extern const _odp_pool_inline_offset_t _odp_pool_inline; - _ODP_INLINE void *odp_packet_data(odp_packet_t pkt) { return _odp_pkt_get(pkt, void *, seg_data); diff --git a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h index 08fb07a62..2f8e0a709 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h @@ -42,8 +42,6 @@ #define _ODP_INLINE #endif -extern const _odp_event_vector_inline_offset_t _odp_event_vector_inline; - _ODP_INLINE odp_packet_vector_t odp_packet_vector_from_event(odp_event_t ev) { _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET_VECTOR); diff --git a/platform/linux-generic/include/odp/api/plat/pool_inline_types.h b/platform/linux-generic/include/odp/api/plat/pool_inline_types.h index 9deec89a1..02f59f982 100644 --- a/platform/linux-generic/include/odp/api/plat/pool_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/pool_inline_types.h @@ -17,25 +17,28 @@ extern "C" { #endif -/** @internal Pool field accessor */ +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +/** Pool field accessor */ #define _odp_pool_get(pool, cast, field) \ (*(cast *)(uintptr_t)((uint8_t *)pool + _odp_pool_inline.field)) -/** @internal Pool header field offsets for inline functions */ +/** Pool header field offsets for inline functions */ typedef struct _odp_pool_inline_offset_t { - /** @internal field offset */ uint16_t index; - /** @internal field offset */ uint16_t seg_len; - /** @internal field offset */ uint16_t uarea_size; - /** @internal field offset */ uint16_t ext_head_offset; - /** @internal field offset */ uint16_t ext_pkt_buf_size; } _odp_pool_inline_offset_t; +extern const _odp_pool_inline_offset_t _odp_pool_inline; + +/** @endcond */ + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp/api/plat/pool_inlines.h b/platform/linux-generic/include/odp/api/plat/pool_inlines.h index a304d2881..58d66fad2 100644 --- a/platform/linux-generic/include/odp/api/plat/pool_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/pool_inlines.h @@ -19,8 +19,6 @@ extern "C" { /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -extern const _odp_pool_inline_offset_t _odp_pool_inline; - #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline diff --git a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h index 4eade3dea..e957785c4 100644 --- a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h @@ -56,6 +56,8 @@ typedef struct { } _odp_queue_api_fn_t; +extern _odp_queue_inline_offset_t _odp_queue_inline_offset; + /** @endcond */ #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp/api/plat/queue_inlines.h b/platform/linux-generic/include/odp/api/plat/queue_inlines.h index c557b4ba3..22673a887 100644 --- a/platform/linux-generic/include/odp/api/plat/queue_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/queue_inlines.h @@ -11,7 +11,6 @@ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -extern _odp_queue_inline_offset_t _odp_queue_inline_offset; extern const _odp_queue_api_fn_t *_odp_queue_api; #ifndef _ODP_NO_INLINE diff --git a/platform/linux-generic/include/odp/api/plat/timer_inline_types.h b/platform/linux-generic/include/odp/api/plat/timer_inline_types.h index ec6804c72..330cbe4ce 100644 --- a/platform/linux-generic/include/odp/api/plat/timer_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/timer_inline_types.h @@ -29,6 +29,8 @@ typedef struct _odp_timeout_inline_offset_t { } _odp_timeout_inline_offset_t; +extern const _odp_timeout_inline_offset_t _odp_timeout_inline_offset; + /** @endcond */ #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp/api/plat/timer_inlines.h b/platform/linux-generic/include/odp/api/plat/timer_inlines.h index 4f5620caf..648459c78 100644 --- a/platform/linux-generic/include/odp/api/plat/timer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/timer_inlines.h @@ -17,8 +17,6 @@ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -extern const _odp_timeout_inline_offset_t _odp_timeout_inline_offset; - #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline diff --git a/platform/linux-generic/include/odp_classification_datamodel.h b/platform/linux-generic/include/odp_classification_datamodel.h index 00df57bef..daa783b07 100644 --- a/platform/linux-generic/include/odp_classification_datamodel.h +++ b/platform/linux-generic/include/odp_classification_datamodel.h @@ -145,7 +145,9 @@ typedef struct ODP_ALIGNED_CACHE cos_s { bool queue_group; odp_cls_hash_proto_t hash_proto; odp_pktin_vector_config_t vector; /* Packet vector config */ +#if ODP_DEPRECATED_API odp_cls_drop_t drop_policy; /* Associated Drop Policy */ +#endif size_t headroom; /* Headroom for this CoS */ odp_spinlock_t lock; /* cos lock */ odp_queue_param_t queue_param; diff --git a/platform/linux-generic/include/odp_ipsec_internal.h b/platform/linux-generic/include/odp_ipsec_internal.h index 8e0cb925e..571796691 100644 --- a/platform/linux-generic/include/odp_ipsec_internal.h +++ b/platform/linux-generic/include/odp_ipsec_internal.h @@ -83,10 +83,13 @@ int _odp_ipsec_status_send(odp_queue_t queue, int result, odp_ipsec_warn_t warn); -#define IPSEC_MAX_IV_LEN 32 /**< Maximum IV length in bytes */ +#define IPSEC_MAX_IV_LEN 16 /**< Maximum cipher IV length in bytes */ #define IPSEC_MAX_SALT_LEN 4 /**< Maximum salt length in bytes */ +#define CBC_SALT_LEN 8 +#define CBC_IV_LEN (CBC_SALT_LEN + sizeof(uint64_t)) + #define IPSEC_SEQ_HI_LEN 4 /**< ESN Higher bits length in bytes */ /* The minimum supported AR window size */ @@ -167,7 +170,10 @@ struct ipsec_sa_s { uint32_t esp_iv_len; uint32_t esp_pad_mask; - uint8_t salt[IPSEC_MAX_SALT_LEN]; + union { + uint8_t salt[IPSEC_MAX_SALT_LEN]; + uint8_t cbc_salt[CBC_SALT_LEN]; + }; uint32_t salt_length; odp_ipsec_lookup_mode_t lookup_mode; @@ -186,6 +192,7 @@ struct ipsec_sa_s { /* Only for outbound */ unsigned use_counter_iv : 1; + unsigned use_cbc_iv : 1; unsigned tun_ipv4 : 1; /* Only for inbound */ @@ -272,6 +279,14 @@ struct ipsec_sa_s { } out; }; } sa_info; + + /* + * Flag to check if the SA soft expiry status event was already + * sent. This field is applicable only for the soft expiry status + * event that gets generated for IPsec SAs configured in inline + * outbound mode. + */ + odp_atomic_u32_t soft_expiry_notified; }; /** diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index 844088ac1..954602959 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -39,7 +39,6 @@ extern "C" { #include <string.h> #include <sys/select.h> -#define PKTIO_MAX_QUEUES ODP_PKTOUT_MAX_QUEUES #define PKTIO_LSO_PROFILES 16 /* Assume at least Ethernet header per each segment */ #define PKTIO_LSO_MIN_PAYLOAD_OFFSET 14 @@ -70,9 +69,9 @@ struct pktio_if_ops; #elif defined(_ODP_PKTIO_XDP) #define PKTIO_PRIVATE_SIZE 29696 #elif defined(_ODP_PKTIO_DPDK) && ODP_CACHE_LINE_SIZE == 128 -#define PKTIO_PRIVATE_SIZE 4032 +#define PKTIO_PRIVATE_SIZE 4160 #elif defined(_ODP_PKTIO_DPDK) -#define PKTIO_PRIVATE_SIZE 3840 +#define PKTIO_PRIVATE_SIZE 3968 #else #define PKTIO_PRIVATE_SIZE 384 #endif @@ -158,21 +157,13 @@ typedef struct ODP_ALIGNED_CACHE { odp_queue_t queue; odp_pktin_queue_t pktin; odp_pktin_vector_config_t vector; - } in_queue[PKTIO_MAX_QUEUES]; + } in_queue[ODP_PKTIN_MAX_QUEUES]; struct { odp_queue_t queue; odp_pktout_queue_t pktout; - } out_queue[PKTIO_MAX_QUEUES]; + } out_queue[ODP_PKTOUT_MAX_QUEUES]; - /**< inotify instance for pcapng fifos */ - struct { - enum { - PCAPNG_WR_STOP = 0, - PCAPNG_WR_PKT, - } state[PKTIO_MAX_QUEUES]; - int fd[PKTIO_MAX_QUEUES]; - } pcapng; } pktio_entry_t; typedef struct { diff --git a/platform/linux-generic/include/odp_pcapng.h b/platform/linux-generic/include/odp_pcapng.h index b88427238..6f2a3dda5 100644 --- a/platform/linux-generic/include/odp_pcapng.h +++ b/platform/linux-generic/include/odp_pcapng.h @@ -19,8 +19,8 @@ extern "C" { int _odp_pcapng_start(pktio_entry_t *entry); void _odp_pcapng_stop(pktio_entry_t *entry); -int _odp_pcapng_write_pkts(pktio_entry_t *entry, int qidx, - const odp_packet_t packets[], int num); +int _odp_pcapng_dump_pkts(pktio_entry_t *entry, int qidx, + const odp_packet_t packets[], int num); #ifdef __cplusplus } diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index f1291ddcd..96ac4c640 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -137,7 +137,9 @@ void odp_cls_cos_param_init(odp_cls_cos_param_t *param) param->queue = ODP_QUEUE_INVALID; param->pool = ODP_POOL_INVALID; +#if ODP_DEPRECATED_API param->drop_policy = ODP_COS_DROP_NEVER; +#endif param->num_queue = 1; param->vector.enable = false; odp_queue_param_init(¶m->queue_param); @@ -234,9 +236,11 @@ static inline void _cls_queue_unwind(uint32_t tbl_index, uint32_t j) odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param_in) { +#if ODP_DEPRECATED_API + odp_cls_drop_t drop_policy; +#endif uint32_t i, j; odp_queue_t queue; - odp_cls_drop_t drop_policy; cos_t *cos; uint32_t tbl_index; odp_cls_cos_param_t param = *param_in; @@ -275,7 +279,9 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param_ } } +#if ODP_DEPRECATED_API drop_policy = param.drop_policy; +#endif for (i = 0; i < CLS_COS_MAX_ENTRY; i++) { cos = &cos_tbl->cos_entry[i]; @@ -336,7 +342,9 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param_ cos->pool = param.pool; cos->headroom = 0; cos->valid = 1; +#if ODP_DEPRECATED_API cos->drop_policy = drop_policy; +#endif odp_atomic_init_u32(&cos->num_rule, 0); cos->index = i; cos->vector = param.vector; @@ -493,6 +501,8 @@ uint32_t odp_cls_cos_queues(odp_cos_t cos_id, odp_queue_t queue[], return cos->num_queue; } +#if ODP_DEPRECATED_API + int odp_cos_drop_set(odp_cos_t cos_id, odp_cls_drop_t drop_policy) { cos_t *cos = get_cos_entry(cos_id); @@ -519,6 +529,8 @@ odp_cls_drop_t odp_cos_drop(odp_cos_t cos_id) return cos->drop_policy; } +#endif + int odp_pktio_default_cos_set(odp_pktio_t pktio_in, odp_cos_t default_cos) { pktio_entry_t *entry; @@ -581,10 +593,8 @@ int odp_pktio_headroom_set(odp_pktio_t pktio_in, uint32_t headroom) return 0; } -int odp_cos_with_l2_priority(odp_pktio_t pktio_in, - uint8_t num_qos, - uint8_t qos_table[], - odp_cos_t cos_table[]) +int ODP_DEPRECATE(odp_cos_with_l2_priority)(odp_pktio_t pktio_in, uint8_t num_qos, + uint8_t qos_table[], odp_cos_t cos_table[]) { pmr_l2_cos_t *l2_cos; uint32_t i; diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 6be1a01ad..28d46c794 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -9,8 +9,9 @@ #include <odp/api/ipsec.h> #include <odp/api/chksum.h> -#include <odp/api/plat/packet_inlines.h> #include <odp/api/plat/byteorder_inlines.h> +#include <odp/api/plat/ipsec_inlines.h> +#include <odp/api/plat/packet_inlines.h> #include <odp/api/plat/queue_inlines.h> #include <odp_global_data.h> @@ -550,7 +551,13 @@ static int ipsec_in_iv(odp_packet_t pkt, ipsec_sa_t *ipsec_sa, uint16_t iv_offset) { - memcpy(state->iv, ipsec_sa->salt, ipsec_sa->salt_length); + if (ipsec_sa->salt_length > 0) { + /* It is faster to just copy MAX_SALT_LEN bytes than the exact length */ + ODP_STATIC_ASSERT(IPSEC_MAX_SALT_LEN <= IPSEC_MAX_IV_LEN, + "IPSEC_MAX_SALT_LEN too large"); + memcpy(state->iv, ipsec_sa->salt, IPSEC_MAX_SALT_LEN); + } + _ODP_ASSERT(ipsec_sa->salt_length + ipsec_sa->esp_iv_len <= IPSEC_MAX_IV_LEN); if (odp_packet_copy_to_mem(pkt, iv_offset, ipsec_sa->esp_iv_len, @@ -558,6 +565,7 @@ static int ipsec_in_iv(odp_packet_t pkt, return -1; if (ipsec_sa->aes_ctr_iv) { + ODP_STATIC_ASSERT(IPSEC_MAX_IV_LEN >= 16, "IPSEC_MAX_IV_LEN too small"); state->iv[12] = 0; state->iv[13] = 0; state->iv[14] = 0; @@ -1396,6 +1404,9 @@ static int ipsec_random_data(uint8_t *data, uint32_t len) return 0; } +/* + * Generate cipher IV for outbound processing. + */ static int ipsec_out_iv(ipsec_state_t *state, ipsec_sa_t *ipsec_sa, uint64_t seq_no) @@ -1404,21 +1415,42 @@ static int ipsec_out_iv(ipsec_state_t *state, /* Both GCM and CTR use 8-bit counters */ _ODP_ASSERT(sizeof(seq_no) == ipsec_sa->esp_iv_len); - /* Check for overrun */ - if (seq_no == 0) - return -1; + /* It is faster to just copy MAX_SALT_LEN bytes than the exact length */ + ODP_STATIC_ASSERT(IPSEC_MAX_SALT_LEN <= IPSEC_MAX_IV_LEN, + "IPSEC_MAX_SALT_LEN too large"); + memcpy(state->iv, ipsec_sa->salt, IPSEC_MAX_SALT_LEN); - memcpy(state->iv, ipsec_sa->salt, ipsec_sa->salt_length); - memcpy(state->iv + ipsec_sa->salt_length, &seq_no, - ipsec_sa->esp_iv_len); + _ODP_ASSERT(ipsec_sa->salt_length + sizeof(seq_no) <= IPSEC_MAX_IV_LEN); + memcpy(state->iv + ipsec_sa->salt_length, &seq_no, sizeof(seq_no)); if (ipsec_sa->aes_ctr_iv) { + ODP_STATIC_ASSERT(IPSEC_MAX_IV_LEN >= 16, "IPSEC_MAX_IV_LEN too small"); state->iv[12] = 0; state->iv[13] = 0; state->iv[14] = 0; state->iv[15] = 1; } - } else if (ipsec_sa->esp_iv_len) { + } else if (ipsec_sa->use_cbc_iv) { + /* + * For CBC mode ciphers with 16 byte IV we generate the cipher + * IV by concatenating a per-session random salt value and + * 64-bit sequence number. The ESP IV will be generated at + * ciphering time by CBC-encrypting a zero block using the + * cipher IV. + * + * This way each packet of an SA will have an unpredictable + * IV and different SAs (e.g. manually keyed SAs across + * restarts) will have different IV sequences (so one cannot + * predict IVs of an SA by observing the IVs of another SA + * with the same key). + */ + _ODP_ASSERT(CBC_SALT_LEN + sizeof(seq_no) == ipsec_sa->esp_iv_len); + ODP_STATIC_ASSERT(CBC_SALT_LEN + sizeof(seq_no) <= IPSEC_MAX_IV_LEN, + "IPSEC_MAX_IV_LEN too small for CBC IV construction"); + memcpy(state->iv, ipsec_sa->cbc_salt, CBC_SALT_LEN); + memcpy(state->iv + CBC_SALT_LEN, &seq_no, sizeof(seq_no)); + } else if (odp_unlikely(ipsec_sa->esp_iv_len)) { + _ODP_ASSERT(ipsec_sa->esp_iv_len <= IPSEC_MAX_IV_LEN); if (ipsec_random_data(state->iv, ipsec_sa->esp_iv_len)) return -1; } @@ -1559,10 +1591,13 @@ static int ipsec_out_esp(odp_packet_t *pkt, odp_packet_copy_from_mem(*pkt, ipsec_offset, _ODP_ESPHDR_LEN, &esp); - odp_packet_copy_from_mem(*pkt, - ipsec_offset + _ODP_ESPHDR_LEN, - ipsec_sa->esp_iv_len, - state->iv + ipsec_sa->salt_length); + if (!ipsec_sa->use_cbc_iv) { + /* copy the relevant part of cipher IV to ESP IV */ + odp_packet_copy_from_mem(*pkt, + ipsec_offset + _ODP_ESPHDR_LEN, + ipsec_sa->esp_iv_len, + state->iv + ipsec_sa->salt_length); + } /* 0xa5 is a good value to fill data instead of generating random data * to create TFC padding */ _odp_packet_set_data(*pkt, esptrl_offset - esptrl.pad_len - tfc_len, @@ -1613,6 +1648,21 @@ static int ipsec_out_esp(odp_packet_t *pkt, ipsec_sa->icv_len; state->stats_length = param->cipher_range.length; + + if (ipsec_sa->use_cbc_iv) { + /* + * Encrypt zeroed ESP IV field using the special cipher IV + * to create the final unpredictable ESP IV + */ + _ODP_ASSERT(ipsec_sa->esp_iv_len == CBC_IV_LEN); + param->cipher_range.offset -= CBC_IV_LEN; + param->cipher_range.length += CBC_IV_LEN; + _odp_packet_set_data(*pkt, + ipsec_offset + _ODP_ESPHDR_LEN, + 0, + CBC_IV_LEN); + } + param->session = ipsec_sa->session; return 0; @@ -2477,6 +2527,25 @@ static void ipsec_out_inline_finalize(odp_packet_t pkt_in[], ipsec_inline_op_t *op = &ops[i]; odp_packet_t *pkt = &pkt_in[i]; + if (op->op.status.warn.soft_exp_packets || op->op.status.warn.soft_exp_bytes) { + if (!odp_atomic_load_u32(&op->op.sa->soft_expiry_notified)) { + int rc; + + /* + * Another thread may have sent the notification by now but we do + * not care since sending duplicate expiry notifications is allowed. + */ + rc = _odp_ipsec_status_send(op->op.sa->queue, + ODP_IPSEC_STATUS_WARN, + op->op.sa->ipsec_sa_hdl, + 0, op->op.status.warn); + if (rc == 0) + odp_atomic_store_u32(&op->op.sa->soft_expiry_notified, 1); + else + _ODP_DBG("IPsec status event submission failed\n"); + } + } + if (odp_unlikely(op->op.status.error.all)) goto handle_err; @@ -2535,34 +2604,6 @@ int odp_ipsec_test_sa_update(odp_ipsec_sa_t sa, return 0; } -int odp_ipsec_result(odp_ipsec_packet_result_t *result, odp_packet_t packet) -{ - odp_ipsec_packet_result_t *res; - - _ODP_ASSERT(result != NULL); - - res = ipsec_pkt_result(packet); - - /* FIXME: maybe postprocess here, setting alg error in case of crypto - * error instead of processing packet fully in ipsec_in/out_single */ - - *result = *res; - - return 0; -} - -odp_packet_t odp_ipsec_packet_from_event(odp_event_t ev) -{ - _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); - _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_IPSEC); - return odp_packet_from_event(ev); -} - -odp_event_t odp_ipsec_packet_to_event(odp_packet_t pkt) -{ - return odp_packet_to_event(pkt); -} - int odp_ipsec_stats(odp_ipsec_sa_t sa, odp_ipsec_stats_t *stats) { ipsec_sa_t *ipsec_sa; diff --git a/platform/linux-generic/odp_ipsec_api.c b/platform/linux-generic/odp_ipsec_api.c new file mode 100644 index 000000000..1d1abe84a --- /dev/null +++ b/platform/linux-generic/odp_ipsec_api.c @@ -0,0 +1,11 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/ipsec.h> + +/* Non-inlined versions of API functions */ +#define _ODP_NO_INLINE +#include <odp/api/plat/ipsec_inlines.h> diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 3097bef4b..1b3a90e6a 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -485,6 +485,25 @@ static void store_sa_info(ipsec_sa_t *ipsec_sa, const odp_ipsec_sa_param_t *p) ipsec_sa->sa_info.out.mtu = p->outbound.mtu; } +static int init_cbc_salt(ipsec_sa_t *ipsec_sa) +{ + int filled = 0; + int rc; + + if (!ipsec_sa->use_cbc_iv) + return 0; + + while (filled < CBC_SALT_LEN) { + rc = odp_random_data(&ipsec_sa->cbc_salt[filled], + CBC_SALT_LEN - filled, + ODP_RANDOM_CRYPTO); + if (rc < 0) + return -1; + filled += rc; + } + return 0; +} + odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) { ipsec_sa_t *ipsec_sa; @@ -562,6 +581,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) odp_atomic_init_u64(&ipsec_sa->stats.hard_exp_pkts_err, 0); odp_atomic_init_u64(&ipsec_sa->stats.post_lifetime_err_pkts, 0); odp_atomic_init_u64(&ipsec_sa->stats.post_lifetime_err_bytes, 0); + odp_atomic_init_u32(&ipsec_sa->soft_expiry_notified, 0); if (ODP_IPSEC_MODE_TUNNEL == ipsec_sa->mode && ODP_IPSEC_DIR_OUTBOUND == param->dir) { @@ -655,6 +675,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) ipsec_sa->esp_pad_mask = esp_block_len_to_mask(8); break; case ODP_CIPHER_ALG_AES_CBC: + ipsec_sa->use_cbc_iv = 1; ipsec_sa->esp_iv_len = 16; ipsec_sa->esp_pad_mask = esp_block_len_to_mask(16); break; @@ -745,6 +766,9 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) memcpy(ipsec_sa->salt, salt_param->data, ipsec_sa->salt_length); } + if (init_cbc_salt(ipsec_sa)) + goto error; + if (odp_crypto_session_create(&crypto_param, &ipsec_sa->session, &ses_create_rc)) goto error; diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 2c4f364e3..8bb558be2 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -2174,3 +2174,71 @@ void _odp_ishm_print(int block_index) odp_spinlock_unlock(&ishm_tbl->lock); } + +int32_t odp_system_meminfo(odp_system_meminfo_t *info, odp_system_memblock_t memblock[], + int32_t max_num) +{ + ishm_block_t *block; + int name_len, proc_index; + int32_t i; + uintptr_t addr; + uint64_t len, lost, page_size; + uint64_t lost_total = 0; + uint64_t len_total = 0; + int32_t num = 0; + const uint64_t huge_sz = odp_sys_huge_page_size(); + const uint64_t normal_sz = odp_sys_page_size(); + + odp_spinlock_lock(&ishm_tbl->lock); + procsync(); + + for (i = 0; i < ISHM_MAX_NB_BLOCKS; i++) { + block = &ishm_tbl->block[i]; + + len = block->len; + if (len == 0) + continue; + + lost = len - block->user_len; + + if (num < max_num) { + odp_system_memblock_t *mb = &memblock[num]; + + name_len = strlen(block->name); + if (name_len >= ODP_SYSTEM_MEMBLOCK_NAME_LEN) + name_len = ODP_SYSTEM_MEMBLOCK_NAME_LEN - 1; + + memcpy(mb->name, block->name, name_len); + mb->name[name_len] = 0; + + addr = 0; + proc_index = procfind_block(i); + if (proc_index >= 0) + addr = (uintptr_t)ishm_proctable->entry[proc_index].start; + + page_size = 0; + if (block->huge == HUGE) + page_size = huge_sz; + else if (block->huge == NORMAL) + page_size = normal_sz; + + mb->addr = addr; + mb->used = len; + mb->overhead = lost; + mb->page_size = page_size; + } + + len_total += len; + lost_total += lost; + + num++; + } + + odp_spinlock_unlock(&ishm_tbl->lock); + + info->total_mapped = len_total; + info->total_used = len_total; + info->total_overhead = lost_total; + + return num; +} diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 2ebf602e8..212e48de0 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -67,8 +67,8 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .input_flags = offsetof(odp_packet_hdr_t, p.input_flags), .flags = offsetof(odp_packet_hdr_t, p.flags), .subtype = offsetof(odp_packet_hdr_t, subtype), - .cls_mark = offsetof(odp_packet_hdr_t, cls_mark) - + .cls_mark = offsetof(odp_packet_hdr_t, cls_mark), + .ipsec_ctx = offsetof(odp_packet_hdr_t, ipsec_ctx), }; #include <odp/visibility_end.h> diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 794452127..22b6bc916 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -259,7 +259,7 @@ static void init_out_queues(pktio_entry_t *entry) { int i; - for (i = 0; i < PKTIO_MAX_QUEUES; i++) { + for (i = 0; i < ODP_PKTOUT_MAX_QUEUES; i++) { entry->out_queue[i].queue = ODP_QUEUE_INVALID; entry->out_queue[i].pktout = PKTOUT_INVALID; } @@ -279,7 +279,7 @@ static void init_pktio_entry(pktio_entry_t *entry) odp_atomic_init_u64(&entry->stats_extra.out_discards, 0); odp_atomic_init_u64(&entry->tx_ts, 0); - for (i = 0; i < PKTIO_MAX_QUEUES; i++) { + for (i = 0; i < ODP_PKTIN_MAX_QUEUES; i++) { entry->in_queue[i].queue = ODP_QUEUE_INVALID; entry->in_queue[i].pktin = PKTIN_INVALID; } @@ -602,6 +602,12 @@ int odp_pktio_config(odp_pktio_t hdl, const odp_pktio_config_t *config) return -1; } + if (config->flow_control.pause_rx != ODP_PKTIO_LINK_PAUSE_OFF || + config->flow_control.pause_tx != ODP_PKTIO_LINK_PAUSE_OFF) { + _ODP_ERR("Link flow control is not supported\n"); + return -1; + } + lock_entry(entry); if (entry->state == PKTIO_STATE_STARTED) { unlock_entry(entry); @@ -1509,6 +1515,8 @@ void odp_pktio_config_init(odp_pktio_config_t *config) config->parser.layer = ODP_PROTO_LAYER_ALL; config->reassembly.max_num_frags = 2; + config->flow_control.pause_rx = ODP_PKTIO_LINK_PAUSE_OFF; + config->flow_control.pause_tx = ODP_PKTIO_LINK_PAUSE_OFF; } int odp_pktio_info(odp_pktio_t hdl, odp_pktio_info_t *info) @@ -1813,6 +1821,10 @@ int odp_pktio_capability(odp_pktio_t pktio, odp_pktio_capability_t *capa) capa->reassembly.ip = false; capa->reassembly.ipv4 = false; capa->reassembly.ipv6 = false; + capa->flow_control.pause_rx = 0; + capa->flow_control.pfc_rx = 0; + capa->flow_control.pause_tx = 0; + capa->flow_control.pfc_tx = 0; return ret; } @@ -2172,12 +2184,12 @@ int odp_pktin_queue_config(odp_pktio_t pktio, entry = get_pktio_entry(pktio); if (entry == NULL) { - _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); + _ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } if (entry->state == PKTIO_STATE_STARTED) { - _ODP_DBG("pktio %s: not stopped\n", entry->name); + _ODP_ERR("pktio %s: not stopped\n", entry->name); return -1; } @@ -2188,7 +2200,7 @@ int odp_pktin_queue_config(odp_pktio_t pktio, return 0; if (!param->classifier_enable && param->num_queues == 0) { - _ODP_DBG("invalid num_queues for operation mode\n"); + _ODP_ERR("invalid num_queues for operation mode\n"); return -1; } @@ -2196,17 +2208,39 @@ int odp_pktin_queue_config(odp_pktio_t pktio, rc = odp_pktio_capability(pktio, &capa); if (rc) { - _ODP_DBG("pktio %s: unable to read capabilities\n", entry->name); + _ODP_ERR("pktio %s: unable to read capabilities\n", entry->name); return -1; } entry->enabled.cls = !!param->classifier_enable; if (num_queues > capa.max_input_queues) { - _ODP_DBG("pktio %s: too many input queues\n", entry->name); + _ODP_ERR("pktio %s: too many input queues\n", entry->name); return -1; } + /* Check input queue sizes in direct mode */ + for (i = 0; i < num_queues && mode == ODP_PKTIN_MODE_DIRECT; i++) { + uint32_t queue_size = param->queue_size[i]; + + if (queue_size == 0) + continue; + + if (capa.max_input_queue_size == 0) { + _ODP_ERR("pktio %s: configuring input queue size not supported\n", + entry->name); + return -1; + } + if (queue_size < capa.min_input_queue_size) { + _ODP_ERR("pktio %s: input queue size too small\n", entry->name); + return -1; + } + if (queue_size > capa.max_input_queue_size) { + _ODP_ERR("pktio %s: input queue size too large\n", entry->name); + return -1; + } + } + /* Validate packet vector parameters */ if (param->vector.enable) { odp_pool_t pool = param->vector.pool; @@ -2282,7 +2316,7 @@ int odp_pktin_queue_config(odp_pktio_t pktio, queue = odp_queue_create(name, &queue_param); if (queue == ODP_QUEUE_INVALID) { - _ODP_DBG("pktio %s: event queue create failed\n", entry->name); + _ODP_ERR("pktio %s: event queue create failed\n", entry->name); destroy_in_queues(entry, i + 1); return -1; } @@ -2660,13 +2694,6 @@ int odp_pktout_queue(odp_pktio_t pktio, odp_pktout_queue_t queues[], int num) return num_queues; } -static inline void _odp_dump_pcapng_pkts(pktio_entry_t *entry, int qidx, - const odp_packet_t packets[], int num) -{ - if (odp_unlikely(entry->pcapng.state[qidx] == PCAPNG_WR_PKT)) - _odp_pcapng_write_pkts(entry, qidx, packets, num); -} - int odp_pktin_recv(odp_pktin_queue_t queue, odp_packet_t packets[], int num) { pktio_entry_t *entry; @@ -2684,7 +2711,7 @@ int odp_pktin_recv(odp_pktin_queue_t queue, odp_packet_t packets[], int num) ret = entry->ops->recv(entry, queue.index, packets, num); if (_ODP_PCAPNG) - _odp_dump_pcapng_pkts(entry, queue.index, packets, ret); + _odp_pcapng_dump_pkts(entry, queue.index, packets, ret); return ret; } @@ -2715,7 +2742,7 @@ int odp_pktin_recv_tmo(odp_pktin_queue_t queue, odp_packet_t packets[], int num, ret = entry->ops->recv_tmo(entry, queue.index, packets, num, wait); if (_ODP_PCAPNG) - _odp_dump_pcapng_pkts(entry, queue.index, packets, ret); + _odp_pcapng_dump_pkts(entry, queue.index, packets, ret); return ret; } @@ -2723,7 +2750,7 @@ int odp_pktin_recv_tmo(odp_pktin_queue_t queue, odp_packet_t packets[], int num, while (1) { ret = entry->ops->recv(entry, queue.index, packets, num); if (_ODP_PCAPNG) - _odp_dump_pcapng_pkts(entry, queue.index, packets, ret); + _odp_pcapng_dump_pkts(entry, queue.index, packets, ret); if (ret != 0 || wait == 0) return ret; @@ -2791,8 +2818,7 @@ int odp_pktin_recv_mq_tmo(const odp_pktin_queue_t queues[], uint32_t num_q, uint entry = get_pktio_entry(queues[lfrom].pktio); if (entry) - _odp_dump_pcapng_pkts(entry, lfrom, packets, - ret); + _odp_pcapng_dump_pkts(entry, lfrom, packets, ret); } return ret; @@ -2910,7 +2936,7 @@ int odp_pktout_send(odp_pktout_queue_t queue, const odp_packet_t packets[], return 0; if (_ODP_PCAPNG) - _odp_dump_pcapng_pkts(entry, queue.index, packets, num); + _odp_pcapng_dump_pkts(entry, queue.index, packets, num); if (odp_unlikely(_odp_pktio_tx_compl_enabled(entry))) { for (int i = 0; i < num; i++) diff --git a/platform/linux-generic/odp_pcapng.c b/platform/linux-generic/odp_pcapng.c index d97e1ca9d..4423b0483 100644 --- a/platform/linux-generic/odp_pcapng.c +++ b/platform/linux-generic/odp_pcapng.c @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2019, Nokia + * Copyright (c) 2019-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -11,11 +11,14 @@ #if defined(_ODP_PCAPNG) && _ODP_PCAPNG == 1 +#include <odp/api/hints.h> #include <odp/api/shared_memory.h> #include <odp/api/spinlock.h> #include <odp/api/plat/packet_inlines.h> +#include <odp/api/plat/packet_io_inlines.h> +#include <odp_config_internal.h> #include <odp_global_data.h> #include <odp_init_internal.h> #include <odp_macros_internal.h> @@ -43,6 +46,9 @@ #define INOTIFY_BUF_LEN (16 * (sizeof(struct inotify_event))) #define PCAPNG_WATCH_DIR "/var/run/odp/" +#define PKTIO_MAX_QUEUES (ODP_PKTIN_MAX_QUEUES > ODP_PKTOUT_MAX_QUEUES ? \ + ODP_PKTIN_MAX_QUEUES : ODP_PKTOUT_MAX_QUEUES) + /* pcapng: enhanced packet block file encoding */ typedef struct ODP_PACKED pcapng_section_hdr_block_s { uint32_t block_type; @@ -73,19 +79,36 @@ typedef struct pcapng_enhanced_packet_block_s { uint32_t packet_len; } pcapng_enhanced_packet_block_t; +/** Pktio entry specific data */ +typedef struct { + pktio_entry_t *pktio_entry; + + /* inotify instances for pcapng fifos */ + enum { + PCAPNG_WR_STOP = 0, + PCAPNG_WR_PKT, + } state[PKTIO_MAX_QUEUES]; + int fd[PKTIO_MAX_QUEUES]; +} pcapng_entry_t; + typedef struct ODP_ALIGNED_CACHE { odp_shm_t shm; - pktio_entry_t *entry[ODP_CONFIG_PKTIO_ENTRIES]; int num_entries; pthread_t inotify_thread; int inotify_fd; int inotify_watch_fd; int inotify_is_running; odp_spinlock_t lock; + pcapng_entry_t entry[ODP_CONFIG_PKTIO_ENTRIES]; } pcapng_global_t; static pcapng_global_t *pcapng_gbl; +static inline pcapng_entry_t *pcapng_entry(pktio_entry_t *pktio_entry) +{ + return &pcapng_gbl->entry[odp_pktio_index(pktio_entry->handle)]; +} + int write_pcapng_hdr(pktio_entry_t *entry, int qidx); int _odp_pcapng_init_global(void) @@ -132,6 +155,7 @@ static void pcapng_drain_fifo(int fd) static void inotify_event_handle(pktio_entry_t *entry, int qidx, struct inotify_event *event) { + pcapng_entry_t *pcapng = pcapng_entry(entry); int mtu = _ODP_MAX(odp_pktin_maxlen(entry->handle), odp_pktout_maxlen(entry->handle)); if (event->mask & IN_OPEN) { @@ -140,23 +164,23 @@ static void inotify_event_handle(pktio_entry_t *entry, int qidx, if (PIPE_BUF < mtu + sizeof(pcapng_enhanced_packet_block_t) + sizeof(uint32_t)) { _ODP_ERR("PIPE_BUF:%d too small. Disabling pcap\n", PIPE_BUF); - entry->pcapng.state[qidx] = PCAPNG_WR_STOP; + pcapng->state[qidx] = PCAPNG_WR_STOP; return; } ret = write_pcapng_hdr(entry, qidx); if (ret) { - entry->pcapng.state[qidx] = PCAPNG_WR_STOP; + pcapng->state[qidx] = PCAPNG_WR_STOP; } else { - entry->pcapng.state[qidx] = PCAPNG_WR_PKT; + pcapng->state[qidx] = PCAPNG_WR_PKT; _ODP_DBG("Open %s for pcap tracing\n", event->name); } } else if (event->mask & IN_CLOSE) { - int fd = entry->pcapng.fd[qidx]; + int fd = pcapng->fd[qidx]; pcapng_drain_fifo(fd); - entry->pcapng.state[qidx] = PCAPNG_WR_STOP; + pcapng->state[qidx] = PCAPNG_WR_STOP; _ODP_DBG("Close %s for pcap tracing\n", event->name); } else { _ODP_ERR("Unknown inotify event 0x%08x\n", event->mask); @@ -207,7 +231,7 @@ static pktio_entry_t *pktio_from_event(struct inotify_event *event) odp_spinlock_lock(&pcapng_gbl->lock); for (i = 0; i < ODP_CONFIG_PKTIO_ENTRIES; i++) { - pktio_entry_t *entry = pcapng_gbl->entry[i]; + pktio_entry_t *entry = pcapng_gbl->entry[i].pktio_entry; if (entry == NULL) continue; @@ -287,6 +311,7 @@ static int get_fifo_max_size(void) int _odp_pcapng_start(pktio_entry_t *entry) { + pcapng_entry_t *pcapng = pcapng_entry(entry); int ret = -1, fd; pthread_attr_t attr; unsigned int i; @@ -301,8 +326,8 @@ int _odp_pcapng_start(pktio_entry_t *entry) char pcapng_name[128]; char pcapng_path[256]; - entry->pcapng.fd[i] = -1; - entry->pcapng.state[i] = PCAPNG_WR_STOP; + pcapng->fd[i] = -1; + pcapng->state[i] = PCAPNG_WR_STOP; get_pcapng_fifo_name(pcapng_name, sizeof(pcapng_name), entry->name, i); @@ -320,7 +345,7 @@ int _odp_pcapng_start(pktio_entry_t *entry) fd = open(pcapng_path, O_RDWR | O_NONBLOCK); if (fd == -1) { _ODP_ERR("Fail to open fifo\n"); - entry->pcapng.state[i] = PCAPNG_WR_STOP; + pcapng->state[i] = PCAPNG_WR_STOP; if (remove(pcapng_path) == -1) _ODP_ERR("Can't remove fifo %s\n", pcapng_path); continue; @@ -333,14 +358,14 @@ int _odp_pcapng_start(pktio_entry_t *entry) _ODP_DBG("set pcap fifo size %i\n", fifo_sz); } - entry->pcapng.fd[i] = fd; + pcapng->fd[i] = fd; } odp_spinlock_lock(&pcapng_gbl->lock); /* already running from a previous pktio */ if (pcapng_gbl->inotify_is_running == 1) { - pcapng_gbl->entry[odp_pktio_index(entry->handle)] = entry; + pcapng->pktio_entry = entry; pcapng_gbl->num_entries++; odp_spinlock_unlock(&pcapng_gbl->lock); return 0; @@ -371,7 +396,7 @@ int _odp_pcapng_start(pktio_entry_t *entry) if (ret) { _ODP_ERR("Can't start inotify thread (ret=%d). pcapng disabled.\n", ret); } else { - pcapng_gbl->entry[odp_pktio_index(entry->handle)] = entry; + pcapng->pktio_entry = entry; pcapng_gbl->num_entries++; pcapng_gbl->inotify_is_running = 1; } @@ -390,13 +415,14 @@ out_destroy: void _odp_pcapng_stop(pktio_entry_t *entry) { + pcapng_entry_t *pcapng = pcapng_entry(entry); int ret; unsigned int i; unsigned int max_queue = _ODP_MAX(entry->num_in_queue, entry->num_out_queue); odp_spinlock_lock(&pcapng_gbl->lock); - pcapng_gbl->entry[odp_pktio_index(entry->handle)] = NULL; + pcapng->pktio_entry = NULL; pcapng_gbl->num_entries--; if (pcapng_gbl->inotify_is_running == 1 && @@ -427,8 +453,8 @@ void _odp_pcapng_stop(pktio_entry_t *entry) char pcapng_name[128]; char pcapng_path[256]; - entry->pcapng.state[i] = PCAPNG_WR_STOP; - close(entry->pcapng.fd[i]); + pcapng->state[i] = PCAPNG_WR_STOP; + close(pcapng->fd[i]); get_pcapng_fifo_name(pcapng_name, sizeof(pcapng_name), entry->name, i); @@ -442,10 +468,11 @@ void _odp_pcapng_stop(pktio_entry_t *entry) int write_pcapng_hdr(pktio_entry_t *entry, int qidx) { + pcapng_entry_t *pcapng = pcapng_entry(entry); size_t len; pcapng_section_hdr_block_t shb; pcapng_interface_description_block_t idb; - int fd = entry->pcapng.fd[qidx]; + int fd = pcapng->fd[qidx]; memset(&shb, 0, sizeof(shb)); memset(&idb, 0, sizeof(idb)); @@ -503,17 +530,21 @@ static ssize_t write_fifo(int fd, struct iovec *iov, int iovcnt) return len; } -int _odp_pcapng_write_pkts(pktio_entry_t *entry, int qidx, - const odp_packet_t packets[], int num) +int _odp_pcapng_dump_pkts(pktio_entry_t *entry, int qidx, + const odp_packet_t packets[], int num) { + pcapng_entry_t *pcapng = pcapng_entry(entry); int i = 0; struct iovec packet_iov[3 * num]; pcapng_enhanced_packet_block_t epb[num]; int iovcnt = 0; ssize_t block_len = 0; - int fd = entry->pcapng.fd[qidx]; + int fd = pcapng->fd[qidx]; ssize_t len = 0, wlen; + if (odp_likely(pcapng->state[qidx] != PCAPNG_WR_PKT)) + return 0; + for (i = 0; i < num; i++) { odp_packet_hdr_t *pkt_hdr = packet_hdr(packets[i]); uint32_t seg_len; diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index f90189d05..172a4d336 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -716,7 +716,7 @@ static void pktio_start(int pktio_idx, _ODP_ASSERT(pktio_idx < ODP_CONFIG_PKTIO_ENTRIES); for (i = 0; i < num_in_queue; i++) { rxq = in_queue_idx[i]; - _ODP_ASSERT(rxq < PKTIO_MAX_QUEUES); + _ODP_ASSERT(rxq < ODP_PKTIN_MAX_QUEUES); __atomic_fetch_add(&global->poll_count[pktio_idx], 1, __ATOMIC_RELAXED); qentry = _odp_qentry_from_ext(odpq[i]); diff --git a/platform/linux-generic/odp_shared_memory.c b/platform/linux-generic/odp_shared_memory.c index 4103568a7..a0e822a53 100644 --- a/platform/linux-generic/odp_shared_memory.c +++ b/platform/linux-generic/odp_shared_memory.c @@ -124,6 +124,31 @@ int odp_shm_info(odp_shm_t shm, odp_shm_info_t *info) info->size = ishm_info.size; info->page_size = ishm_info.page_size; info->flags = ishm_info.user_flags; + info->num_seg = 1; + + return 0; +} + +int odp_shm_segment_info(odp_shm_t shm, uint32_t index, uint32_t num, + odp_shm_segment_info_t seg_info[]) +{ + odp_shm_info_t info; + + /* No physical memory segment information available */ + if (index != 0 || num != 1) { + _ODP_ERR("Only single segment supported (%u, %u)\n", index, num); + return -1; + } + + if (odp_shm_info(shm, &info)) { + _ODP_ERR("SHM info call failed\n"); + return -1; + } + + seg_info[0].addr = (uintptr_t)info.addr; + seg_info[0].iova = ODP_SHM_IOVA_INVALID; + seg_info[0].pa = ODP_SHM_PA_INVALID; + seg_info[0].len = info.size; return 0; } diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index 90e83a4f2..05660eb7a 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -652,7 +652,7 @@ static odp_event_t timer_cancel(timer_pool_t *tp, uint32_t idx) static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) { uint64_t exp_tck; - odp_queue_t queue; + odp_queue_t queue = ODP_QUEUE_INVALID; _odp_timer_t *tim = &tp->timers[idx]; tick_buf_t *tb = &tp->tick_buf[idx]; odp_event_t tmo_event = ODP_EVENT_INVALID; diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 30e6a0e61..7e9db90ce 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -104,7 +104,7 @@ ODP_STATIC_ASSERT(DPDK_MIN_RX_BURST <= UINT8_MAX, "DPDK_MIN_RX_BURST too large") /** DPDK runtime configuration options */ typedef struct { - int num_rx_desc; + int num_rx_desc_default; int num_tx_desc_default; uint8_t multicast_en; uint8_t rx_drop_en; @@ -141,7 +141,7 @@ typedef struct ODP_ALIGNED_CACHE { /* Minimum RX burst size */ uint8_t min_rx_burst; /* Cache for storing extra RX packets */ - pkt_cache_t rx_cache[PKTIO_MAX_QUEUES]; + pkt_cache_t rx_cache[ODP_PKTIN_MAX_QUEUES]; /* --- Control path data --- */ @@ -161,15 +161,17 @@ typedef struct ODP_ALIGNED_CACHE { uint8_t mtu_set; /* Use system call to get/set vdev promisc mode */ uint8_t vdev_sysc_promisc; + /* Number of RX descriptors per queue */ + uint16_t num_rx_desc[ODP_PKTIN_MAX_QUEUES]; /* Number of TX descriptors per queue */ - uint16_t num_tx_desc[PKTIO_MAX_QUEUES]; + uint16_t num_tx_desc[ODP_PKTOUT_MAX_QUEUES]; /* --- Locks for MT safe operations --- */ /* RX queue locks */ - odp_ticketlock_t rx_lock[PKTIO_MAX_QUEUES] ODP_ALIGNED_CACHE; + odp_ticketlock_t rx_lock[ODP_PKTIN_MAX_QUEUES] ODP_ALIGNED_CACHE; /* TX queue locks */ - odp_ticketlock_t tx_lock[PKTIO_MAX_QUEUES] ODP_ALIGNED_CACHE; + odp_ticketlock_t tx_lock[ODP_PKTOUT_MAX_QUEUES] ODP_ALIGNED_CACHE; } pkt_dpdk_t; ODP_STATIC_ASSERT(PKTIO_PRIVATE_SIZE >= sizeof(pkt_dpdk_t), @@ -227,14 +229,8 @@ static int init_options(pktio_entry_t *pktio_entry, int val; if (!lookup_opt("num_rx_desc", dev_info->driver_name, - &opt->num_rx_desc)) + &opt->num_rx_desc_default)) return -1; - if (opt->num_rx_desc < dev_info->rx_desc_lim.nb_min || - opt->num_rx_desc > dev_info->rx_desc_lim.nb_max || - opt->num_rx_desc % dev_info->rx_desc_lim.nb_align) { - _ODP_ERR("Invalid number of RX descriptors\n"); - return -1; - } if (!lookup_opt("num_tx_desc", dev_info->driver_name, &opt->num_tx_desc_default)) @@ -255,7 +251,7 @@ static int init_options(pktio_entry_t *pktio_entry, _ODP_DBG("DPDK interface (%s): %" PRIu16 "\n", dev_info->driver_name, pkt_priv(pktio_entry)->port_id); _ODP_DBG(" multicast_en: %d\n", opt->multicast_en); - _ODP_DBG(" num_rx_desc: %d\n", opt->num_rx_desc); + _ODP_DBG(" num_rx_desc: %d\n", opt->num_rx_desc_default); _ODP_DBG(" num_tx_desc: %d\n", opt->num_tx_desc_default); _ODP_DBG(" rx_drop_en: %d\n", opt->rx_drop_en); @@ -1264,7 +1260,7 @@ static int dpdk_close(pktio_entry_t *pktio_entry) unsigned i, j; /* Free cache packets */ - for (i = 0; i < PKTIO_MAX_QUEUES; i++) { + for (i = 0; i < ODP_PKTIN_MAX_QUEUES; i++) { idx = pkt_dpdk->rx_cache[i].idx; for (j = 0; j < pkt_dpdk->rx_cache[i].count; j++) @@ -1493,21 +1489,51 @@ static void prepare_rss_conf(pktio_entry_t *pktio_entry, static int dpdk_input_queues_config(pktio_entry_t *pktio_entry, const odp_pktin_queue_param_t *p) { + struct rte_eth_dev_info dev_info; + pkt_dpdk_t *pkt_dpdk = pkt_priv(pktio_entry); odp_pktin_mode_t mode = pktio_entry->param.in_mode; uint8_t lockless; + int ret; prepare_rss_conf(pktio_entry, p); /** * Scheduler synchronizes input queue polls. Only single thread * at a time polls a queue */ - if (mode == ODP_PKTIN_MODE_SCHED || - p->op_mode == ODP_PKTIO_OP_MT_UNSAFE) + if (mode == ODP_PKTIN_MODE_SCHED || p->op_mode == ODP_PKTIO_OP_MT_UNSAFE) lockless = 1; else lockless = 0; - pkt_priv(pktio_entry)->flags.lockless_rx = lockless; + pkt_dpdk->flags.lockless_rx = lockless; + + ret = rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info); + if (ret) { + _ODP_ERR("DPDK: rte_eth_dev_info_get() failed: %d\n", ret); + return -1; + } + + /* Configure RX descriptors */ + for (uint32_t i = 0; i < p->num_queues; i++) { + uint16_t num_rx_desc = pkt_dpdk->opt.num_rx_desc_default; + + if (mode == ODP_PKTIN_MODE_DIRECT && p->queue_size[i] != 0) { + num_rx_desc = p->queue_size[i]; + /* Make sure correct alignment is used */ + if (dev_info.rx_desc_lim.nb_align) + num_rx_desc = RTE_ALIGN_MUL_CEIL(num_rx_desc, + dev_info.rx_desc_lim.nb_align); + } + + if (num_rx_desc < dev_info.rx_desc_lim.nb_min || + num_rx_desc > dev_info.rx_desc_lim.nb_max || + num_rx_desc % dev_info.rx_desc_lim.nb_align) { + _ODP_ERR("DPDK: invalid number of RX descriptors (%" PRIu16 ") for queue " + "%" PRIu32 "\n", num_rx_desc, i); + return -1; + } + pkt_dpdk->num_rx_desc[i] = num_rx_desc; + } return 0; } @@ -1548,7 +1574,8 @@ static int dpdk_output_queues_config(pktio_entry_t *pktio_entry, if (num_tx_desc < dev_info.tx_desc_lim.nb_min || num_tx_desc > dev_info.tx_desc_lim.nb_max || num_tx_desc % dev_info.tx_desc_lim.nb_align) { - _ODP_ERR("DPDK: invalid number of TX descriptors\n"); + _ODP_ERR("DPDK: invalid number of TX descriptors (%" PRIu16 ") for queue " + "%" PRIu32 "\n", num_tx_desc, i); return -1; } pkt_dpdk->num_tx_desc[i] = num_tx_desc; @@ -1572,7 +1599,9 @@ static int dpdk_init_capability(pktio_entry_t *pktio_entry, memset(capa, 0, sizeof(odp_pktio_capability_t)); capa->max_input_queues = RTE_MIN(dev_info->max_rx_queues, - PKTIO_MAX_QUEUES); + ODP_PKTIN_MAX_QUEUES); + capa->min_input_queue_size = dev_info->rx_desc_lim.nb_min; + capa->max_input_queue_size = dev_info->rx_desc_lim.nb_max; /* ixgbe devices support only 16 rx queues in RSS mode */ if (!strncmp(dev_info->driver_name, IXGBE_DRV_NAME, @@ -1581,7 +1610,7 @@ static int dpdk_init_capability(pktio_entry_t *pktio_entry, capa->max_input_queues); capa->max_output_queues = RTE_MIN(dev_info->max_tx_queues, - PKTIO_MAX_QUEUES); + ODP_PKTOUT_MAX_QUEUES); capa->min_output_queue_size = dev_info->tx_desc_lim.nb_min; capa->max_output_queue_size = dev_info->tx_desc_lim.nb_max; @@ -1845,10 +1874,11 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED, return -1; } - for (i = 0; i < PKTIO_MAX_QUEUES; i++) { + for (i = 0; i < ODP_PKTIN_MAX_QUEUES; i++) odp_ticketlock_init(&pkt_dpdk->rx_lock[i]); + + for (i = 0; i < ODP_PKTOUT_MAX_QUEUES; i++) odp_ticketlock_init(&pkt_dpdk->tx_lock[i]); - } rte_eth_stats_reset(pkt_dpdk->port_id); @@ -1903,7 +1933,7 @@ static int dpdk_setup_eth_rx(const pktio_entry_t *pktio_entry, for (i = 0; i < pktio_entry->num_in_queue; i++) { ret = rte_eth_rx_queue_setup(port_id, i, - pkt_dpdk->opt.num_rx_desc, + pkt_dpdk->num_rx_desc[i], rte_eth_dev_socket_id(port_id), &rxconf, pkt_dpdk->pkt_pool); if (ret < 0) { diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index 947edd476..b30535f22 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -44,12 +44,15 @@ #define LOOP_MTU_MIN 68 #define LOOP_MTU_MAX UINT16_MAX -#define LOOP_MAX_TX_QUEUE_SIZE 1024 +#define LOOP_MAX_QUEUE_SIZE 1024 typedef struct { odp_queue_t loopq; /**< loopback queue for "loop" device */ + uint32_t pktin_queue_size; /**< input queue size */ + uint32_t pktout_queue_size; /**< output queue size */ uint16_t mtu; /**< link MTU */ uint8_t idx; /**< index of "loop" device */ + uint8_t queue_create; /**< create or re-create queue during start */ } pkt_loop_t; ODP_STATIC_ASSERT(PKTIO_PRIVATE_SIZE >= sizeof(pkt_loop_t), @@ -84,9 +87,11 @@ static int loopback_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, return -1; } + memset(pkt_loop, 0, sizeof(pkt_loop_t)); pkt_loop->idx = idx; pkt_loop->mtu = LOOP_MTU_MAX; pkt_loop->loopq = ODP_QUEUE_INVALID; + pkt_loop->queue_create = 1; loopback_stats_reset(pktio_entry); loopback_init_capability(pktio_entry); @@ -112,19 +117,23 @@ static int loopback_queue_destroy(odp_queue_t queue) return 0; } -static int loopback_pktout_queue_config(pktio_entry_t *pktio_entry, - const odp_pktout_queue_param_t *param) +static int loopback_start(pktio_entry_t *pktio_entry) { pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); odp_queue_param_t queue_param; char queue_name[ODP_QUEUE_NAME_LEN]; + /* Re-create queue only when necessary */ + if (!pkt_loop->queue_create) + return 0; + /* Destroy old queue */ if (pkt_loop->loopq != ODP_QUEUE_INVALID && loopback_queue_destroy(pkt_loop->loopq)) return -1; odp_queue_param_init(&queue_param); - queue_param.size = param->queue_size[0]; + queue_param.size = pkt_loop->pktin_queue_size > pkt_loop->pktout_queue_size ? + pkt_loop->pktin_queue_size : pkt_loop->pktout_queue_size; snprintf(queue_name, sizeof(queue_name), "_odp_pktio_loopq-%" PRIu64 "", odp_pktio_to_u64(pktio_entry->handle)); @@ -134,6 +143,31 @@ static int loopback_pktout_queue_config(pktio_entry_t *pktio_entry, _ODP_ERR("Creating loopback pktio queue failed\n"); return -1; } + pkt_loop->queue_create = 0; + + return 0; +} + +static int loopback_pktin_queue_config(pktio_entry_t *pktio_entry, + const odp_pktin_queue_param_t *param) +{ + pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); + + if (pktio_entry->param.in_mode == ODP_PKTIN_MODE_DIRECT) { + pkt_loop->pktin_queue_size = param->queue_size[0]; + pkt_loop->queue_create = 1; + } + + return 0; +} + +static int loopback_pktout_queue_config(pktio_entry_t *pktio_entry, + const odp_pktout_queue_param_t *param) +{ + pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); + + pkt_loop->pktout_queue_size = param->queue_size[0]; + pkt_loop->queue_create = 1; return 0; } @@ -485,10 +519,15 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry) capa->maxlen.min_output = LOOP_MTU_MIN; capa->maxlen.max_output = LOOP_MTU_MAX; + capa->min_input_queue_size = 1; + capa->max_input_queue_size = queue_capa.plain.max_size; + if (capa->max_input_queue_size == 0) + capa->max_input_queue_size = LOOP_MAX_QUEUE_SIZE; + capa->min_output_queue_size = 1; capa->max_output_queue_size = queue_capa.plain.max_size; if (capa->max_output_queue_size == 0) - capa->max_output_queue_size = LOOP_MAX_TX_QUEUE_SIZE; + capa->max_output_queue_size = LOOP_MAX_QUEUE_SIZE; odp_pktio_config_init(&capa->config); capa->config.enable_loop = 1; @@ -591,7 +630,7 @@ const pktio_if_ops_t _odp_loopback_pktio_ops = { .term = NULL, .open = loopback_open, .close = loopback_close, - .start = NULL, + .start = loopback_start, .stop = NULL, .stats = loopback_stats, .stats_reset = loopback_stats_reset, @@ -612,6 +651,6 @@ const pktio_if_ops_t _odp_loopback_pktio_ops = { .pktio_ts_from_ns = NULL, .pktio_time = NULL, .config = NULL, - .input_queues_config = NULL, + .input_queues_config = loopback_pktin_queue_config, .output_queues_config = loopback_pktout_queue_config, }; diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index 41ded2ae1..0352e33f7 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -95,9 +95,9 @@ typedef struct { odp_bool_t lockless_rx; /**< no locking for rx */ odp_bool_t lockless_tx; /**< no locking for tx */ /** mapping of pktin queues to netmap rx descriptors */ - netmap_ring_t rx_desc_ring[PKTIO_MAX_QUEUES]; + netmap_ring_t rx_desc_ring[ODP_PKTIN_MAX_QUEUES]; /** mapping of pktout queues to netmap tx descriptors */ - netmap_ring_t tx_desc_ring[PKTIO_MAX_QUEUES]; + netmap_ring_t tx_desc_ring[ODP_PKTOUT_MAX_QUEUES]; netmap_opt_t opt; /**< options */ } pkt_netmap_t; @@ -314,13 +314,15 @@ static inline void netmap_close_descriptors(pktio_entry_t *pktio_entry) int i, j; pkt_netmap_t *pkt_nm = pkt_priv(pktio_entry); - for (i = 0; i < PKTIO_MAX_QUEUES; i++) { + for (i = 0; i < ODP_PKTIN_MAX_QUEUES; i++) { for (j = 0; j < NM_MAX_DESC; j++) { if (pkt_nm->rx_desc_ring[i].desc[j] != NULL) { nm_close(pkt_nm->rx_desc_ring[i].desc[j]); pkt_nm->rx_desc_ring[i].desc[j] = NULL; } } + } + for (i = 0; i < ODP_PKTOUT_MAX_QUEUES; i++) { for (j = 0; j < NM_MAX_DESC; j++) { if (pkt_nm->tx_desc_ring[i].desc[j] != NULL) { nm_close(pkt_nm->tx_desc_ring[i].desc[j]); @@ -422,8 +424,8 @@ static void netmap_init_capability(pktio_entry_t *pktio_entry) memset(capa, 0, sizeof(odp_pktio_capability_t)); - capa->max_input_queues = PKTIO_MAX_QUEUES; - if (pkt_nm->num_rx_rings < PKTIO_MAX_QUEUES) + capa->max_input_queues = ODP_PKTIN_MAX_QUEUES; + if (pkt_nm->num_rx_rings < ODP_PKTIN_MAX_QUEUES) capa->max_input_queues = pkt_nm->num_rx_rings; if (capa->max_input_queues > NM_MAX_DESC) { /* Have to use a single descriptor to fetch packets from all @@ -434,8 +436,8 @@ static void netmap_init_capability(pktio_entry_t *pktio_entry) NM_MAX_DESC, capa->max_input_queues); } - capa->max_output_queues = PKTIO_MAX_QUEUES; - if (pkt_nm->num_tx_rings < PKTIO_MAX_QUEUES) + capa->max_output_queues = ODP_PKTOUT_MAX_QUEUES; + if (pkt_nm->num_tx_rings < ODP_PKTOUT_MAX_QUEUES) capa->max_output_queues = pkt_nm->num_tx_rings; if (capa->max_output_queues > NM_MAX_DESC) { capa->max_output_queues = NM_MAX_DESC; @@ -578,10 +580,11 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, nm_close(desc); - for (i = 0; i < PKTIO_MAX_QUEUES; i++) { + for (i = 0; i < ODP_PKTIN_MAX_QUEUES; i++) odp_ticketlock_init(&pkt_nm->rx_desc_ring[i].lock); + + for (i = 0; i < ODP_PKTOUT_MAX_QUEUES; i++) odp_ticketlock_init(&pkt_nm->tx_desc_ring[i].lock); - } if (pkt_nm->is_virtual) { static unsigned int mac; diff --git a/platform/linux-generic/pktio/null.c b/platform/linux-generic/pktio/null.c index 0bd33f517..b067a68ec 100644 --- a/platform/linux-generic/pktio/null.c +++ b/platform/linux-generic/pktio/null.c @@ -130,8 +130,8 @@ static int null_capability(pktio_entry_t *pktio_entry ODP_UNUSED, { memset(capa, 0, sizeof(odp_pktio_capability_t)); - capa->max_input_queues = PKTIO_MAX_QUEUES; - capa->max_output_queues = PKTIO_MAX_QUEUES; + capa->max_input_queues = ODP_PKTIN_MAX_QUEUES; + capa->max_output_queues = ODP_PKTOUT_MAX_QUEUES; capa->set_op.op.promisc_mode = 0; odp_pktio_config_init(&capa->config); diff --git a/platform/linux-generic/pktio/socket_xdp.c b/platform/linux-generic/pktio/socket_xdp.c index b10fc9fce..be79ca267 100644 --- a/platform/linux-generic/pktio/socket_xdp.c +++ b/platform/linux-generic/pktio/socket_xdp.c @@ -94,7 +94,7 @@ typedef struct { } xdp_umem_info_t; typedef struct { - xdp_sock_t qs[PKTIO_MAX_QUEUES]; + xdp_sock_t qs[ODP_PKTOUT_MAX_QUEUES]; xdp_umem_info_t *umem_info; uint32_t num_q; int pktio_idx; @@ -275,7 +275,7 @@ static int sock_xdp_open(odp_pktio_t pktio, pktio_entry_t *pktio_entry, const ch priv->max_mtu = pool->seg_len; - for (int i = 0; i < PKTIO_MAX_QUEUES; ++i) { + for (int i = 0; i < ODP_PKTOUT_MAX_QUEUES; ++i) { odp_ticketlock_init(&priv->qs[i].rx_lock); odp_ticketlock_init(&priv->qs[i].tx_lock); } @@ -852,8 +852,8 @@ static int set_queue_capability(int fd, const char *devname, odp_pktio_capabilit channels.max_combined = 1U; } - max_channels = _ODP_MIN((uint32_t)PKTIO_MAX_QUEUES, channels.max_combined); - capa->max_input_queues = max_channels; + max_channels = _ODP_MIN((uint32_t)ODP_PKTOUT_MAX_QUEUES, channels.max_combined); + capa->max_input_queues = _ODP_MIN((uint32_t)ODP_PKTIN_MAX_QUEUES, max_channels); capa->max_output_queues = max_channels; return 0; |