diff options
Diffstat (limited to 'platform/linux-generic')
40 files changed, 733 insertions, 301 deletions
diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 2181abd26..e654381e6 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -293,7 +293,7 @@ __LIB__libodp_linux_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/default/odp_hash_crc32.c \ arch/aarch64/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ + arch/aarch64/odp/api/abi/cpu_time.h \ arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_time.h b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_time.h new file mode 100644 index 000000000..781ee683c --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_time.h @@ -0,0 +1,50 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_CPU_TIME_H_ +#define ODP_API_ABI_CPU_TIME_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +static inline uint64_t _odp_cpu_global_time(void) +{ + uint64_t cntvct; + + __asm__ volatile("mrs %0, cntvct_el0" : "=r"(cntvct) : : "memory"); + + return cntvct; +} + +static inline uint64_t _odp_cpu_global_time_strict(void) +{ + uint64_t cntvct; + + __asm__ volatile("isb" ::: "memory"); + __asm__ volatile("mrs %0, cntvct_el0" : "=r"(cntvct) : : "memory"); + + return cntvct; +} + +static inline uint64_t _odp_cpu_global_time_freq(void) +{ + uint64_t cntfrq; + + __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(cntfrq) : : ); + + return cntfrq; +} + +int _odp_cpu_has_global_time(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp_global_time.c b/platform/linux-generic/arch/aarch64/odp_global_time.c index fa59f5a15..53561b00c 100644 --- a/platform/linux-generic/arch/aarch64/odp_global_time.c +++ b/platform/linux-generic/arch/aarch64/odp_global_time.c @@ -4,33 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#include <odp_posix_extensions.h> - -#include <time.h> - -#include <odp_debug_internal.h> #include <odp/api/abi/cpu_time.h> -#include <odp/visibility_begin.h> - -uint64_t _odp_cpu_global_time(void) -{ - uint64_t cntvct; - - /* - * To be consistent with other architectures, do not issue a - * serializing instruction, e.g. ISB, before reading this - * sys reg. - */ - - /* Memory clobber to minimize optimization around load from sys reg. */ - __asm__ volatile("mrs %0, cntvct_el0" : "=r"(cntvct) : : "memory"); - - return cntvct; -} - -#include <odp/visibility_end.h> - int _odp_cpu_has_global_time(void) { uint64_t hz = _odp_cpu_global_time_freq(); @@ -48,12 +23,3 @@ int _odp_cpu_has_global_time(void) */ return hz >= 1000000 && hz <= 6000000000; } - -uint64_t _odp_cpu_global_time_freq(void) -{ - uint64_t cntfrq; - - __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(cntfrq) : : ); - - return cntfrq; -} diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h b/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h index 09138e70a..24e1c7d33 100644 --- a/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h +++ b/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h @@ -15,6 +15,7 @@ extern "C" { int _odp_cpu_has_global_time(void); uint64_t _odp_cpu_global_time(void); +uint64_t _odp_cpu_global_time_strict(void); uint64_t _odp_cpu_global_time_freq(void); #ifdef __cplusplus diff --git a/platform/linux-generic/arch/default/odp_global_time.c b/platform/linux-generic/arch/default/odp_global_time.c index facffea7d..ee835413f 100644 --- a/platform/linux-generic/arch/default/odp_global_time.c +++ b/platform/linux-generic/arch/default/odp_global_time.c @@ -13,6 +13,11 @@ uint64_t _odp_cpu_global_time(void) return 0; } +uint64_t _odp_cpu_global_time_strict(void) +{ + return 0; +} + #include <odp/visibility_end.h> int _odp_cpu_has_global_time(void) diff --git a/platform/linux-generic/arch/x86/odp/api/abi/cpu_time.h b/platform/linux-generic/arch/x86/odp/api/abi/cpu_time.h index 05ff0db94..c74c4d606 100644 --- a/platform/linux-generic/arch/x86/odp/api/abi/cpu_time.h +++ b/platform/linux-generic/arch/x86/odp/api/abi/cpu_time.h @@ -19,6 +19,12 @@ static inline uint64_t _odp_cpu_global_time(void) return _odp_cpu_rdtsc(); } +static inline uint64_t _odp_cpu_global_time_strict(void) +{ + __atomic_thread_fence(__ATOMIC_SEQ_CST); + return _odp_cpu_rdtsc(); +} + int _odp_cpu_has_global_time(void); uint64_t _odp_cpu_global_time_freq(void); diff --git a/platform/linux-generic/include/odp/api/plat/time_inlines.h b/platform/linux-generic/include/odp/api/plat/time_inlines.h index 1ca495065..bb2913532 100644 --- a/platform/linux-generic/include/odp/api/plat/time_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/time_inlines.h @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2020, Nokia + * Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -46,6 +46,18 @@ static inline odp_time_t _odp_time_cur(void) return _odp_timespec_cur(); } +static inline odp_time_t _odp_time_cur_strict(void) +{ + if (_odp_time_glob.use_hw) { + odp_time_t time; + + time.count = _odp_cpu_global_time_strict() - _odp_time_glob.hw_start; + return time; + } + + return _odp_timespec_cur(); +} + static inline uint64_t _odp_time_hw_to_ns(odp_time_t time) { uint64_t nsec; @@ -79,6 +91,12 @@ static inline uint64_t _odp_time_convert_to_ns(odp_time_t time) #define odp_time_to_ns __odp_time_to_ns #define odp_time_local_ns __odp_time_local_ns #define odp_time_global_ns __odp_time_global_ns + + #define odp_time_local_strict __odp_time_local_strict + #define odp_time_global_strict __odp_time_global_strict + #define odp_time_local_strict_ns __odp_time_local_strict_ns + #define odp_time_global_strict_ns __odp_time_global_strict_ns + #define odp_time_cmp __odp_time_cmp #define odp_time_diff __odp_time_diff #define odp_time_sum __odp_time_sum @@ -97,9 +115,14 @@ _ODP_INLINE odp_time_t odp_time_global(void) return _odp_time_cur(); } -_ODP_INLINE uint64_t odp_time_to_ns(odp_time_t time) +_ODP_INLINE odp_time_t odp_time_local_strict(void) { - return _odp_time_convert_to_ns(time); + return _odp_time_cur_strict(); +} + +_ODP_INLINE odp_time_t odp_time_global_strict(void) +{ + return _odp_time_cur_strict(); } _ODP_INLINE uint64_t odp_time_local_ns(void) @@ -112,6 +135,21 @@ _ODP_INLINE uint64_t odp_time_global_ns(void) return _odp_time_convert_to_ns(_odp_time_cur()); } +_ODP_INLINE uint64_t odp_time_local_strict_ns(void) +{ + return _odp_time_convert_to_ns(_odp_time_cur_strict()); +} + +_ODP_INLINE uint64_t odp_time_global_strict_ns(void) +{ + return _odp_time_convert_to_ns(_odp_time_cur_strict()); +} + +_ODP_INLINE uint64_t odp_time_to_ns(odp_time_t time) +{ + return _odp_time_convert_to_ns(time); +} + _ODP_INLINE int odp_time_cmp(odp_time_t t2, odp_time_t t1) { if (odp_likely(t2.u64 > t1.u64)) diff --git a/platform/linux-generic/include/odp_debug_internal.h b/platform/linux-generic/include/odp_debug_internal.h index dfad95b19..5027cfe4a 100644 --- a/platform/linux-generic/include/odp_debug_internal.h +++ b/platform/linux-generic/include/odp_debug_internal.h @@ -33,8 +33,15 @@ extern "C" { * level 0 to N. */ #define CONFIG_DEBUG_LEVEL 0 +ODP_PRINTF_FORMAT(1, 2) +static inline void check_printf_format(const char *fmt, ...) +{ + (void)fmt; +} + #define _ODP_LOG_FN(level, fmt, ...) \ do { \ + check_printf_format(fmt, ##__VA_ARGS__); \ if (_odp_this_thread && _odp_this_thread->log_fn) \ _odp_this_thread->log_fn(level, fmt, ##__VA_ARGS__); \ else \ diff --git a/platform/linux-generic/include/odp_ipsec_internal.h b/platform/linux-generic/include/odp_ipsec_internal.h index 2509d22ab..96153007f 100644 --- a/platform/linux-generic/include/odp_ipsec_internal.h +++ b/platform/linux-generic/include/odp_ipsec_internal.h @@ -1,4 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2018, 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -169,7 +170,7 @@ struct ipsec_sa_s { struct { odp_ipsec_frag_mode_t frag_mode; - uint32_t mtu; + odp_atomic_u32_t mtu; union { struct { @@ -196,13 +197,15 @@ struct ipsec_sa_s { odp_atomic_u64_t hard_exp_pkts_err; /* - * Track error packets after lifetime check is done. + * Track error packets and bytes after lifetime check is done. * Required since, the stats tracking lifetime is being * used for SA success packets stats. */ odp_atomic_u64_t post_lifetime_err_pkts; + odp_atomic_u64_t post_lifetime_err_bytes; } stats; + uint32_t next_sa; odp_ipsec_sa_param_t param; }; @@ -302,10 +305,16 @@ uint16_t _odp_ipsec_sa_alloc_ipv4_id(ipsec_sa_t *ipsec_sa); int _odp_ipsec_try_inline(odp_packet_t *pkt); /** - * Get number of packets successfully processed by the SA + * Populate number of packets and bytes of data successfully processed by the SA + * in the odp_ipsec_stats_t structure passed. * */ -uint64_t _odp_ipsec_sa_stats_pkts(ipsec_sa_t *sa); +void _odp_ipsec_sa_stats_pkts(ipsec_sa_t *sa, odp_ipsec_stats_t *stats); + +/** + * Return true if IPsec operates in sync mode in the given direction. + */ +odp_bool_t _odp_ipsec_is_sync_mode(odp_ipsec_dir_t dir); /** * @} diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index 7bbd78ab0..c5a51d11d 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -36,6 +36,7 @@ extern "C" { #include <net/if.h> #include <linux/if_ether.h> #include <sys/select.h> +#include <inttypes.h> #define PKTIO_MAX_QUEUES 64 #define PKTIO_LSO_PROFILES 16 @@ -237,7 +238,7 @@ static inline pktio_entry_t *get_pktio_entry(odp_pktio_t pktio) return NULL; if (odp_unlikely(_odp_typeval(pktio) > ODP_CONFIG_PKTIO_ENTRIES)) { - ODP_DBG("pktio limit %d/%d exceed\n", + ODP_DBG("pktio limit %" PRIuPTR "/%d exceed\n", _odp_typeval(pktio), ODP_CONFIG_PKTIO_ENTRIES); return NULL; } @@ -302,6 +303,10 @@ int _odp_sock_recv_mq_tmo_try_int_driven(const struct odp_pktin_queue_t queues[] uint64_t usecs, int *trial_successful); +/* Setup PKTOUT with single queue for TM */ +int _odp_pktio_pktout_tm_config(odp_pktio_t pktio_hdl, + odp_pktout_queue_t *queue, bool reconf); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index 535d43608..01593baea 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -45,6 +45,8 @@ AC_CONFIG_FILES([platform/linux-generic/Makefile platform/linux-generic/test/example/Makefile platform/linux-generic/test/example/classifier/Makefile platform/linux-generic/test/example/generator/Makefile + platform/linux-generic/test/example/ipsec_api/Makefile + platform/linux-generic/test/example/ipsec_crypto/Makefile platform/linux-generic/test/example/l2fwd_simple/Makefile platform/linux-generic/test/example/l3fwd/Makefile platform/linux-generic/test/example/packet/Makefile diff --git a/platform/linux-generic/m4/odp_libconfig.m4 b/platform/linux-generic/m4/odp_libconfig.m4 index 7a0b45497..ecfb28b7f 100644 --- a/platform/linux-generic/m4/odp_libconfig.m4 +++ b/platform/linux-generic/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [15]) +m4_define([_odp_config_version_minor], [16]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-generic/odp_comp.c b/platform/linux-generic/odp_comp.c index 8e1de69c2..685c9098a 100644 --- a/platform/linux-generic/odp_comp.c +++ b/platform/linux-generic/odp_comp.c @@ -133,7 +133,7 @@ static void process_input(odp_packet_t pkt_out, do { out_data = odp_packet_offset(pkt_out, start, &out_len, &cur_seg); - ODP_DBG("out_data 0x%x seg_data_ptr 0x%x out_len %d seg 0x%x\n", + ODP_DBG("out_data %p seg_data_ptr %p out_len %d seg %p\n", out_data, odp_packet_seg_data(pkt_out, cur_seg), out_len, cur_seg); @@ -258,7 +258,7 @@ static int deflate_comp(odp_packet_t pkt_in, read, &in_len, &in_seg); - ODP_DBG("data 0x%x in_len %d seg 0x%x len %d\n", + ODP_DBG("data %p in_len %d seg %p len %d\n", data, in_len, in_seg, len); if (in_len > len) diff --git a/platform/linux-generic/odp_fdserver.c b/platform/linux-generic/odp_fdserver.c index 9e0d75de3..3103ddd5c 100644 --- a/platform/linux-generic/odp_fdserver.c +++ b/platform/linux-generic/odp_fdserver.c @@ -246,10 +246,14 @@ static int get_socket(void) int len; /* construct the named socket path: */ - snprintf(sockpath, FDSERVER_SOCKPATH_MAXLEN, FDSERVER_SOCK_FORMAT, - odp_global_ro.shm_dir, - odp_global_ro.uid, - odp_global_ro.main_pid); + len = snprintf(sockpath, FDSERVER_SOCKPATH_MAXLEN, FDSERVER_SOCK_FORMAT, + odp_global_ro.shm_dir, odp_global_ro.uid, + odp_global_ro.main_pid); + + if (len >= FDSERVER_SOCKPATH_MAXLEN || len >= (int)sizeof(remote.sun_path)) { + ODP_ERR("path too long\n"); + return -1; + } s_sock = socket(AF_UNIX, SOCK_STREAM, 0); if (s_sock == -1) { @@ -561,7 +565,7 @@ int _odp_fdserver_init_global(void) int sock; struct sockaddr_un local; pid_t server_pid; - int res; + int len, res; snprintf(sockpath, FDSERVER_SOCKPATH_MAXLEN, FDSERVER_SOCKDIR_FORMAT, odp_global_ro.shm_dir, @@ -570,10 +574,14 @@ int _odp_fdserver_init_global(void) mkdir(sockpath, 0744); /* construct the server named socket path: */ - snprintf(sockpath, FDSERVER_SOCKPATH_MAXLEN, FDSERVER_SOCK_FORMAT, - odp_global_ro.shm_dir, - odp_global_ro.uid, - odp_global_ro.main_pid); + len = snprintf(sockpath, FDSERVER_SOCKPATH_MAXLEN, FDSERVER_SOCK_FORMAT, + odp_global_ro.shm_dir, odp_global_ro.uid, + odp_global_ro.main_pid); + + if (len >= FDSERVER_SOCKPATH_MAXLEN || len >= (int)sizeof(local.sun_path)) { + ODP_ERR("path too long\n"); + return -1; + } /* create UNIX domain socket: */ sock = socket(AF_UNIX, SOCK_STREAM, 0); diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 5379a23d0..ed19392c9 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -19,6 +19,7 @@ #include <odp_ipsec_internal.h> #include <odp/api/plat/queue_inlines.h> #include <odp_classification_internal.h> +#include <odp_libconfig_internal.h> #include <protocols/eth.h> #include <protocols/ip.h> @@ -28,9 +29,42 @@ #include <errno.h> #include <string.h> +typedef enum { + IPSEC_ORDERING_NONE = 0, + IPSEC_ORDERING_SIMPLE, +} ordering_mode_t; + +typedef struct { + ordering_mode_t inbound_ordering_mode; + ordering_mode_t outbound_ordering_mode; + odp_ipsec_config_t ipsec_config; +} ipsec_global_t; + +static ipsec_global_t *ipsec_global; + static odp_ipsec_config_t *ipsec_config; /* + * Wait until the ordered scheduling context of this thread corresponds + * to the head of its input queue. Do nothing if ordering is not requested + * or if not holding an ordered context. + */ +static void wait_for_order(ordering_mode_t mode) +{ + if (mode == IPSEC_ORDERING_NONE) + return; + _odp_sched_fn->order_lock(); + /* + * We rely on the unlock being no-op, so let's not even bother + * calling it. Unlock cannot really be anything but a no-op since + * the scheduler cannot let other threads to continue until at + * scheduling context release time. + * + * _odp_sched_fn->order_unlock(); + */ +} + +/* * Set cabability bits for algorithms that are defined for use with IPsec * and for which the IPsec crypto or auth capability function returns * at least one supported instance. @@ -240,6 +274,14 @@ int odp_ipsec_config(const odp_ipsec_config_t *config) return 0; } +odp_bool_t _odp_ipsec_is_sync_mode(odp_ipsec_dir_t dir) +{ + return ((dir == ODP_IPSEC_DIR_INBOUND && + ipsec_config->inbound_mode == ODP_IPSEC_OP_MODE_SYNC) || + (dir == ODP_IPSEC_DIR_OUTBOUND && + ipsec_config->outbound_mode == ODP_IPSEC_OP_MODE_SYNC)); +} + static odp_ipsec_packet_result_t *ipsec_pkt_result(odp_packet_t packet) { ODP_ASSERT(ODP_EVENT_PACKET_IPSEC == @@ -409,7 +451,7 @@ static inline ipsec_sa_t *ipsec_get_sa(odp_ipsec_sa_t sa, return NULL; } } else { - ipsec_sa = _odp_ipsec_sa_use(sa); + ipsec_sa = _odp_ipsec_sa_entry_from_hdl(sa); ODP_ASSERT(NULL != ipsec_sa); if (ipsec_sa->proto != proto || ipsec_sa->spi != spi) { @@ -710,6 +752,7 @@ ipsec_sa_err_stats_update(ipsec_sa_t *sa, odp_ipsec_op_status_t *status) static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, odp_ipsec_sa_t sa, odp_packet_t *pkt_out, + odp_bool_t enqueue_op, odp_ipsec_op_status_t *status) { ipsec_state_t state; @@ -796,10 +839,15 @@ static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, goto exit; } - if (_odp_ipsec_sa_replay_update(ipsec_sa, - state.in.seq_no, - status) < 0) - goto exit; + if (ipsec_sa->antireplay) { + if (enqueue_op) + wait_for_order(ipsec_global->inbound_ordering_mode); + + if (_odp_ipsec_sa_replay_update(ipsec_sa, + state.in.seq_no, + status) < 0) + goto exit; + } if (_odp_ipsec_sa_lifetime_update(ipsec_sa, state.stats_length, @@ -908,8 +956,11 @@ static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, goto exit; post_lifetime_err_cnt_update: - if (ipsec_config->stats_en) + if (ipsec_config->stats_en) { odp_atomic_inc_u64(&ipsec_sa->stats.post_lifetime_err_pkts); + odp_atomic_add_u64(&ipsec_sa->stats.post_lifetime_err_bytes, + state.stats_length); + } exit: *pkt_out = pkt; @@ -1159,6 +1210,7 @@ static int ipsec_out_esp(odp_packet_t *pkt, odp_crypto_packet_op_param_t *param, odp_ipsec_op_status_t *status, uint32_t mtu, + odp_bool_t enqueue_op, const odp_ipsec_out_opt_t *opt) { _odp_esphdr_t esp; @@ -1204,6 +1256,8 @@ static int ipsec_out_esp(odp_packet_t *pkt, return -1; } + if (enqueue_op) + wait_for_order(ipsec_global->outbound_ordering_mode); seq_no = ipsec_seq_no(ipsec_sa); if (ipsec_out_iv(state, ipsec_sa, seq_no) < 0) { @@ -1331,7 +1385,8 @@ static int ipsec_out_ah(odp_packet_t *pkt, ipsec_sa_t *ipsec_sa, odp_crypto_packet_op_param_t *param, odp_ipsec_op_status_t *status, - uint32_t mtu) + uint32_t mtu, + odp_bool_t enqueue_op) { _odp_ahhdr_t ah; unsigned hdr_len = _ODP_AHHDR_LEN + ipsec_sa->esp_iv_len + @@ -1345,6 +1400,8 @@ static int ipsec_out_ah(odp_packet_t *pkt, return -1; } + if (enqueue_op) + wait_for_order(ipsec_global->outbound_ordering_mode); seq_no = ipsec_seq_no(ipsec_sa); memset(&ah, 0, sizeof(ah)); @@ -1489,6 +1546,7 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, odp_ipsec_sa_t sa, odp_packet_t *pkt_out, const odp_ipsec_out_opt_t *opt, + odp_bool_t enqueue_op, odp_ipsec_op_status_t *status) { ipsec_state_t state; @@ -1533,7 +1591,7 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, frag_mode = opt->flag.frag_mode ? opt->frag_mode : ipsec_sa->out.frag_mode; if (frag_mode == ODP_IPSEC_FRAG_CHECK) - mtu = ipsec_sa->out.mtu; + mtu = odp_atomic_load_u32(&ipsec_sa->out.mtu); else mtu = UINT32_MAX; @@ -1594,9 +1652,10 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, if (ODP_IPSEC_ESP == ipsec_sa->proto) { rc = ipsec_out_esp(&pkt, &state, ipsec_sa, ¶m, status, mtu, - opt); + enqueue_op, opt); } else if (ODP_IPSEC_AH == ipsec_sa->proto) { - rc = ipsec_out_ah(&pkt, &state, ipsec_sa, ¶m, status, mtu); + rc = ipsec_out_ah(&pkt, &state, ipsec_sa, ¶m, status, mtu, + enqueue_op); } else { status->error.alg = 1; goto exit; @@ -1652,8 +1711,11 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, goto exit; post_lifetime_err_cnt_update: - if (ipsec_config->stats_en) + if (ipsec_config->stats_en) { odp_atomic_inc_u64(&ipsec_sa->stats.post_lifetime_err_pkts); + odp_atomic_add_u64(&ipsec_sa->stats.post_lifetime_err_bytes, + state.stats_length); + } exit: *pkt_out = pkt; @@ -1686,7 +1748,7 @@ int odp_ipsec_in(const odp_packet_t pkt_in[], int num_in, ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); } - ipsec_sa = ipsec_in_single(pkt, sa, &pkt, &status); + ipsec_sa = ipsec_in_single(pkt, sa, &pkt, false, &status); packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); result = ipsec_pkt_result(pkt); @@ -1702,8 +1764,12 @@ int odp_ipsec_in(const odp_packet_t pkt_in[], int num_in, out_pkt++; sa_idx += sa_inc; - /* Last thing */ - if (NULL != ipsec_sa) + /* + * We need to decrease SA use count only if the SA was not + * provided to us by the caller but was found through our own + * SA lookup that increased the use count. + */ + if (sa == ODP_IPSEC_SA_INVALID && ipsec_sa) _odp_ipsec_sa_unuse(ipsec_sa); } @@ -1746,7 +1812,7 @@ int odp_ipsec_out(const odp_packet_t pkt_in[], int num_in, else opt = ¶m->opt[opt_idx]; - ipsec_sa = ipsec_out_single(pkt, sa, &pkt, opt, &status); + ipsec_sa = ipsec_out_single(pkt, sa, &pkt, opt, false, &status); ODP_ASSERT(NULL != ipsec_sa); packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); @@ -1794,7 +1860,7 @@ int odp_ipsec_in_enq(const odp_packet_t pkt_in[], int num_in, ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); } - ipsec_sa = ipsec_in_single(pkt, sa, &pkt, &status); + ipsec_sa = ipsec_in_single(pkt, sa, &pkt, true, &status); packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); result = ipsec_pkt_result(pkt); @@ -1815,8 +1881,12 @@ int odp_ipsec_in_enq(const odp_packet_t pkt_in[], int num_in, in_pkt++; sa_idx += sa_inc; - /* Last thing */ - if (NULL != ipsec_sa) + /* + * We need to decrease SA use count only if the SA was not + * provided to us by the caller but was found through our own + * SA lookup that increased the use count. + */ + if (sa == ODP_IPSEC_SA_INVALID && ipsec_sa) _odp_ipsec_sa_unuse(ipsec_sa); } @@ -1853,7 +1923,7 @@ int odp_ipsec_out_enq(const odp_packet_t pkt_in[], int num_in, else opt = ¶m->opt[opt_idx]; - ipsec_sa = ipsec_out_single(pkt, sa, &pkt, opt, &status); + ipsec_sa = ipsec_out_single(pkt, sa, &pkt, opt, true, &status); ODP_ASSERT(NULL != ipsec_sa); packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); @@ -1890,7 +1960,8 @@ int _odp_ipsec_try_inline(odp_packet_t *pkt) memset(&status, 0, sizeof(status)); - ipsec_sa = ipsec_in_single(*pkt, ODP_IPSEC_SA_INVALID, pkt, &status); + ipsec_sa = ipsec_in_single(*pkt, ODP_IPSEC_SA_INVALID, pkt, false, + &status); /* * Route packet back in case of lookup failure or early error before * lookup @@ -1977,7 +2048,7 @@ int odp_ipsec_out_inline(const odp_packet_t pkt_in[], int num_in, else opt = ¶m->opt[opt_idx]; - ipsec_sa = ipsec_out_single(pkt, sa, &pkt, opt, &status); + ipsec_sa = ipsec_out_single(pkt, sa, &pkt, opt, true, &status); ODP_ASSERT(NULL != ipsec_sa); offset = odp_packet_l3_offset(pkt); @@ -2110,7 +2181,7 @@ int odp_ipsec_stats(odp_ipsec_sa_t sa, odp_ipsec_stats_t *stats) ipsec_sa = _odp_ipsec_sa_entry_from_hdl(sa); ODP_ASSERT(NULL != ipsec_sa); - stats->success = _odp_ipsec_sa_stats_pkts(ipsec_sa); + _odp_ipsec_sa_stats_pkts(ipsec_sa, stats); stats->proto_err = odp_atomic_load_u64(&ipsec_sa->stats.proto_err); stats->auth_err = odp_atomic_load_u64(&ipsec_sa->stats.auth_err); stats->antireplay_err = odp_atomic_load_u64(&ipsec_sa->stats.antireplay_err); @@ -2122,6 +2193,27 @@ int odp_ipsec_stats(odp_ipsec_sa_t sa, odp_ipsec_stats_t *stats) return 0; } +static int read_config_file(ipsec_global_t *global) +{ + const char *str_i = "ipsec.ordering.async_inbound"; + const char *str_o = "ipsec.ordering.async_outbound"; + int val; + + if (!_odp_libconfig_lookup_int(str_i, &val)) { + ODP_ERR("Config option '%s' not found.\n", str_i); + return -1; + } + global->inbound_ordering_mode = val; + + if (!_odp_libconfig_lookup_int(str_o, &val)) { + ODP_ERR("Config option '%s' not found.\n", str_o); + return -1; + } + global->outbound_ordering_mode = val; + + return 0; +} + int _odp_ipsec_init_global(void) { odp_shm_t shm; @@ -2129,17 +2221,25 @@ int _odp_ipsec_init_global(void) if (odp_global_ro.disable.ipsec) return 0; - shm = odp_shm_reserve("_odp_ipsec", sizeof(odp_ipsec_config_t), + shm = odp_shm_reserve("_odp_ipsec", sizeof(*ipsec_global), ODP_CACHE_LINE_SIZE, 0); - - ipsec_config = odp_shm_addr(shm); - - if (ipsec_config == NULL) { + if (shm == ODP_SHM_INVALID) { ODP_ERR("Shm reserve failed for odp_ipsec\n"); return -1; } + ipsec_global = odp_shm_addr(shm); + if (ipsec_global == NULL) { + ODP_ERR("ipsec: odp_shm_addr() failed\n"); + odp_shm_free(shm); + return -1; + } + memset(ipsec_global, 0, sizeof(*ipsec_global)); + ipsec_config = &ipsec_global->ipsec_config; - odp_ipsec_config_init(ipsec_config); + if (read_config_file(ipsec_global)) { + odp_shm_free(shm); + return -1; + } memset(&default_out_opt, 0, sizeof(default_out_opt)); diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index da8232b01..d0367bf63 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -22,9 +22,24 @@ #include <string.h> -#define IPSEC_SA_STATE_DISABLE 0x40000000 -#define IPSEC_SA_STATE_FREE 0xc0000000 -#define IPSEC_SA_STATE_RESERVED 0x80000000 +/* + * SA state consists of state value in the high order bits of ipsec_sa_t::state + * and use counter in the low order bits. + * + * An SA cannot be destroyed if its use count is higher than one. Use counter + * is needed for the case SA lookup is done by us and not the application. + * In the latter case we rely on the fact that the application may not pass + * the SA as a parameter to an IPsec operation concurrently with a call + * to odp_ipsec_sa_disable(). + * + * SAs that are free or being disabled cannot be found in SA lookup by ODP. + */ +#define IPSEC_SA_STATE_ACTIVE 0x00000000 /* SA is in use */ +#define IPSEC_SA_STATE_DISABLE 0x40000000 /* SA is being disabled */ +#define IPSEC_SA_STATE_FREE 0xc0000000 /* SA is unused and free */ +#define IPSEC_SA_STATE_MASK 0xc0000000 /* mask of state bits */ + +#define SA_IDX_NONE UINT32_MAX /* * We do not have global IPv4 ID counter that is accessed for every outbound @@ -68,7 +83,7 @@ typedef struct sa_thread_local_s { * Bytes that can be processed in this thread before looking at * the SA-global byte counter and checking hard and soft limits. */ - uint32_t byte_quota; + odp_atomic_u32_t byte_quota; /* * Life time status when this thread last checked the global * counter(s). @@ -88,6 +103,10 @@ typedef struct ipsec_sa_table_t { ring_mpmc_t ipv4_id_ring; uint32_t ipv4_id_data[IPV4_ID_RING_SIZE] ODP_ALIGNED_CACHE; } hot; + struct { + uint32_t head; + odp_spinlock_t lock; + } sa_freelist; uint32_t max_num_sa; odp_shm_t shm; ipsec_thread_local_t per_thread[]; @@ -130,7 +149,7 @@ static void init_sa_thread_local(ipsec_sa_t *sa) for (n = 0; n < thread_count_max; n++) { sa_tl = &ipsec_sa_tbl->per_thread[n].sa[sa->ipsec_sa_idx]; odp_atomic_init_u32(&sa_tl->packet_quota, 0); - sa_tl->byte_quota = 0; + odp_atomic_init_u32(&sa_tl->byte_quota, 0); sa_tl->lifetime_status.all = 0; } } @@ -199,10 +218,15 @@ int _odp_ipsec_sad_init_global(void) ipsec_sa->ipsec_sa_hdl = ipsec_sa_index_to_handle(i); ipsec_sa->ipsec_sa_idx = i; + ipsec_sa->next_sa = i + 1; + if (i == ipsec_sa_tbl->max_num_sa - 1) + ipsec_sa->next_sa = SA_IDX_NONE; odp_atomic_init_u32(&ipsec_sa->state, IPSEC_SA_STATE_FREE); odp_atomic_init_u64(&ipsec_sa->hot.bytes, 0); odp_atomic_init_u64(&ipsec_sa->hot.packets, 0); } + ipsec_sa_tbl->sa_freelist.head = 0; + odp_spinlock_init(&ipsec_sa_tbl->sa_freelist.lock); return 0; } @@ -245,31 +269,32 @@ uint32_t _odp_ipsec_max_num_sa(void) static ipsec_sa_t *ipsec_sa_reserve(void) { - uint32_t i; - ipsec_sa_t *ipsec_sa; - - for (i = 0; i < ipsec_sa_tbl->max_num_sa; i++) { - uint32_t state = IPSEC_SA_STATE_FREE; - - ipsec_sa = ipsec_sa_entry(i); - - if (odp_atomic_cas_acq_u32(&ipsec_sa->state, &state, - IPSEC_SA_STATE_RESERVED)) - return ipsec_sa; + ipsec_sa_t *ipsec_sa = NULL; + uint32_t sa_idx; + + odp_spinlock_lock(&ipsec_sa_tbl->sa_freelist.lock); + sa_idx = ipsec_sa_tbl->sa_freelist.head; + if (sa_idx != SA_IDX_NONE) { + ipsec_sa = ipsec_sa_entry(sa_idx); + ipsec_sa_tbl->sa_freelist.head = ipsec_sa->next_sa; } - - return NULL; + odp_spinlock_unlock(&ipsec_sa_tbl->sa_freelist.lock); + return ipsec_sa; } static void ipsec_sa_release(ipsec_sa_t *ipsec_sa) { - odp_atomic_store_rel_u32(&ipsec_sa->state, IPSEC_SA_STATE_FREE); + odp_spinlock_lock(&ipsec_sa_tbl->sa_freelist.lock); + ipsec_sa->next_sa = ipsec_sa_tbl->sa_freelist.head; + ipsec_sa_tbl->sa_freelist.head = ipsec_sa->ipsec_sa_idx; + odp_atomic_store_u32(&ipsec_sa->state, IPSEC_SA_STATE_FREE); + odp_spinlock_unlock(&ipsec_sa_tbl->sa_freelist.lock); } /* Mark reserved SA as available now */ static void ipsec_sa_publish(ipsec_sa_t *ipsec_sa) { - odp_atomic_store_rel_u32(&ipsec_sa->state, 0); + odp_atomic_store_rel_u32(&ipsec_sa->state, IPSEC_SA_STATE_ACTIVE); } static int ipsec_sa_lock(ipsec_sa_t *ipsec_sa) @@ -280,11 +305,9 @@ static int ipsec_sa_lock(ipsec_sa_t *ipsec_sa) while (0 == cas) { /* * This can be called from lookup path, so we really need this - * check. Thanks to the way flags are defined we actually test - * that the SA is not DISABLED, FREE or RESERVED using just one - * condition. + * check. */ - if (state & IPSEC_SA_STATE_FREE) + if ((state & IPSEC_SA_STATE_MASK) != IPSEC_SA_STATE_ACTIVE) return -1; cas = odp_atomic_cas_acq_u32(&ipsec_sa->state, &state, @@ -440,6 +463,10 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) ipsec_sa->spi = param->spi; ipsec_sa->context = param->context; ipsec_sa->queue = param->dest_queue; + if (_odp_ipsec_is_sync_mode(param->dir)) { + /* Invalid queue indicates sync mode */ + ipsec_sa->queue = ODP_QUEUE_INVALID; + } ipsec_sa->mode = param->mode; ipsec_sa->flags = 0; if (param->opt.esn) { @@ -469,7 +496,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) ipsec_sa->lookup_mode = ODP_IPSEC_LOOKUP_DISABLED; odp_atomic_init_u64(&ipsec_sa->hot.out.seq, 1); ipsec_sa->out.frag_mode = param->outbound.frag_mode; - ipsec_sa->out.mtu = param->outbound.mtu; + odp_atomic_init_u32(&ipsec_sa->out.mtu, param->outbound.mtu); } ipsec_sa->dec_ttl = param->opt.dec_ttl; ipsec_sa->copy_dscp = param->opt.copy_dscp; @@ -492,6 +519,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) odp_atomic_init_u64(&ipsec_sa->stats.hard_exp_bytes_err, 0); odp_atomic_init_u64(&ipsec_sa->stats.hard_exp_pkts_err, 0); odp_atomic_init_u64(&ipsec_sa->stats.post_lifetime_err_pkts, 0); + odp_atomic_init_u64(&ipsec_sa->stats.post_lifetime_err_bytes, 0); /* Copy application provided parameter values. */ ipsec_sa->param = *param; @@ -770,13 +798,9 @@ int odp_ipsec_sa_mtu_update(odp_ipsec_sa_t sa, uint32_t mtu) { ipsec_sa_t *ipsec_sa; - ipsec_sa = _odp_ipsec_sa_use(sa); + ipsec_sa = ipsec_sa_entry_from_hdl(sa); ODP_ASSERT(NULL != ipsec_sa); - - ipsec_sa->out.mtu = mtu; - - _odp_ipsec_sa_unuse(ipsec_sa); - + odp_atomic_store_u32(&ipsec_sa->out.mtu, mtu); return 0; } @@ -835,6 +859,7 @@ int _odp_ipsec_sa_lifetime_update(ipsec_sa_t *ipsec_sa, uint32_t len, { sa_thread_local_t *sa_tl = ipsec_sa_thread_local(ipsec_sa); uint64_t packets, bytes; + uint32_t tl_byte_quota; uint32_t tl_pkt_quota; tl_pkt_quota = odp_atomic_load_u32(&sa_tl->packet_quota); @@ -855,11 +880,12 @@ int _odp_ipsec_sa_lifetime_update(ipsec_sa_t *ipsec_sa, uint32_t len, tl_pkt_quota--; odp_atomic_store_u32(&sa_tl->packet_quota, tl_pkt_quota); - if (odp_unlikely(sa_tl->byte_quota < len)) { + tl_byte_quota = odp_atomic_load_u32(&sa_tl->byte_quota); + if (odp_unlikely(tl_byte_quota < len)) { bytes = odp_atomic_fetch_add_u64(&ipsec_sa->hot.bytes, len + SA_LIFE_BYTES_PREALLOC); bytes += len + SA_LIFE_BYTES_PREALLOC; - sa_tl->byte_quota += len + SA_LIFE_BYTES_PREALLOC; + tl_byte_quota += len + SA_LIFE_BYTES_PREALLOC; if (ipsec_sa->soft_limit_bytes > 0 && bytes >= ipsec_sa->soft_limit_bytes) @@ -869,7 +895,9 @@ int _odp_ipsec_sa_lifetime_update(ipsec_sa_t *ipsec_sa, uint32_t len, bytes >= ipsec_sa->hard_limit_bytes) sa_tl->lifetime_status.error.hard_exp_bytes = 1; } - sa_tl->byte_quota -= len; + tl_byte_quota -= len; + odp_atomic_store_u32(&sa_tl->byte_quota, tl_byte_quota); + status->all |= sa_tl->lifetime_status.all; @@ -909,9 +937,6 @@ int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, int cas = 0; uint64_t state, new_state; - if (!ipsec_sa->antireplay) - return 0; - state = odp_atomic_load_u64(&ipsec_sa->hot.in.antireplay); while (0 == cas) { @@ -973,9 +998,10 @@ uint16_t _odp_ipsec_sa_alloc_ipv4_id(ipsec_sa_t *ipsec_sa) return tl->next_ipv4_id++; } -uint64_t _odp_ipsec_sa_stats_pkts(ipsec_sa_t *sa) +void _odp_ipsec_sa_stats_pkts(ipsec_sa_t *sa, odp_ipsec_stats_t *stats) { int thread_count_max = odp_thread_count_max(); + uint64_t tl_byte_quota = 0; uint64_t tl_pkt_quota = 0; sa_thread_local_t *sa_tl; int n; @@ -994,11 +1020,20 @@ uint64_t _odp_ipsec_sa_stats_pkts(ipsec_sa_t *sa) for (n = 0; n < thread_count_max; n++) { sa_tl = &ipsec_sa_tbl->per_thread[n].sa[sa->ipsec_sa_idx]; tl_pkt_quota += odp_atomic_load_u32(&sa_tl->packet_quota); + tl_byte_quota += odp_atomic_load_u32(&sa_tl->byte_quota); } - return odp_atomic_load_u64(&sa->hot.packets) + stats->success = + odp_atomic_load_u64(&sa->hot.packets) - odp_atomic_load_u64(&sa->stats.post_lifetime_err_pkts) - tl_pkt_quota; + + stats->success_bytes = + odp_atomic_load_u64(&sa->hot.bytes) + - odp_atomic_load_u64(&sa->stats.post_lifetime_err_bytes) + - tl_byte_quota; + + return; } static void ipsec_out_sa_info(ipsec_sa_t *ipsec_sa, odp_ipsec_sa_info_t *sa_info) diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 0014f3c34..7041cd2b5 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -1995,7 +1995,7 @@ int _odp_ishm_status(const char *title) } ODP_PRINT("%s\n", title); - ODP_PRINT(" %-*s flag %-29s %-08s %-08s %-3s %-3s %-3s file\n", + ODP_PRINT(" %-*s flag %-29s %-8s %-8s %-3s %-3s %-3s file\n", max_name_len, "name", "range", "user_len", "unused", "seq", "ref", "fd"); @@ -2045,8 +2045,8 @@ int _odp_ishm_status(const char *title) entry_fd = ishm_proctable->entry[proc_index].fd; } - ODP_PRINT("%2i %-*s %s%c %p-%p %-08" PRIu64 " " - "%-08" PRIu64 " %-3" PRIu64 " %-3" PRIu64 " " + ODP_PRINT("%2i %-*s %s%c %p-%p %-8" PRIu64 " " + "%-8" PRIu64 " %-3" PRIu64 " %-3" PRIu64 " " "%-3d %s\n", i, max_name_len, ishm_tbl->block[i].name, flags, huge, start_addr, end_addr, @@ -2059,7 +2059,7 @@ int _odp_ishm_status(const char *title) ishm_tbl->block[i].filename : "(none)"); } - ODP_PRINT("TOTAL: %58s%-08" PRIu64 " %2s%-08" PRIu64 "\n", + ODP_PRINT("TOTAL: %58s%-8" PRIu64 " %2s%-8" PRIu64 "\n", "", len_total, "", lost_total); ODP_PRINT("%65s(%" PRIu64 "MB) %4s(%" PRIu64 "MB)\n", @@ -2072,15 +2072,15 @@ int _odp_ishm_status(const char *title) fragmnt; fragmnt = fragmnt->next) { if (fragmnt->block_index >= 0) { nb_allocated_frgments++; - ODP_PRINT(" %08p - %08p: ALLOCATED by block:%d\n", - (uintptr_t)fragmnt->start, - (uintptr_t)fragmnt->start + fragmnt->len - 1, + ODP_PRINT(" %8p - %8p: ALLOCATED by block:%d\n", + fragmnt->start, + (void *)((uintptr_t)fragmnt->start + fragmnt->len - 1), fragmnt->block_index); consecutive_unallocated = 0; } else { - ODP_PRINT(" %08p - %08p: NOT ALLOCATED\n", - (uintptr_t)fragmnt->start, - (uintptr_t)fragmnt->start + fragmnt->len - 1); + ODP_PRINT(" %8p - %8p: NOT ALLOCATED\n", + fragmnt->start, + (void *)((uintptr_t)fragmnt->start + fragmnt->len - 1)); if (consecutive_unallocated++) ODP_ERR("defragmentation error\n"); } diff --git a/platform/linux-generic/odp_ishmpool.c b/platform/linux-generic/odp_ishmpool.c index 4b593a4bc..818b0a132 100644 --- a/platform/linux-generic/odp_ishmpool.c +++ b/platform/linux-generic/odp_ishmpool.c @@ -280,7 +280,7 @@ static void *_odp_ishmbud_alloc(pool_t *bpool, uint64_t size) uintptr_t nr; /* if size is zero or too big reject: */ - if ((!size) && (size > (1U << bpool->ctrl.order))) { + if ((!size) && (size > (1ULL << bpool->ctrl.order))) { ODP_ERR("Invalid alloc size (0 or larger than whole pool)\n"); return NULL; } diff --git a/platform/linux-generic/odp_name_table.c b/platform/linux-generic/odp_name_table.c index 3697bc877..9c1814cae 100644 --- a/platform/linux-generic/odp_name_table.c +++ b/platform/linux-generic/odp_name_table.c @@ -376,6 +376,7 @@ static void name_tbl_entry_free(name_tbl_entry_t *name_tbl_entry) memset(name_tbl_entry, 0, sizeof(name_tbl_entry_t)); name_tbl_entry->next_entry = name_tbl->free_list_head; name_tbl->free_list_head = name_tbl_entry; + name_tbl_entry->name_tbl_id = name_tbl_id; } static hash_tbl_entry_t make_hash_tbl_entry(name_tbl_entry_t *name_tbl_entry, diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 711e5f8a6..694b0a741 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -377,7 +377,7 @@ static const char *driver_name(odp_pktio_t hdl) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_ERR("pktio entry %d does not exist\n", hdl); + ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return "bad handle"; } @@ -1253,7 +1253,7 @@ static inline uint32_t pktio_maxlen(odp_pktio_t hdl) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", hdl); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return 0; } @@ -1296,7 +1296,7 @@ int odp_pktio_maxlen_set(odp_pktio_t hdl, uint32_t maxlen_input, entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_ERR("Pktio entry %d does not exist\n", hdl); + ODP_ERR("Pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return -1; } @@ -1360,7 +1360,7 @@ int odp_pktio_promisc_mode_set(odp_pktio_t hdl, odp_bool_t enable) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", hdl); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return -1; } @@ -1390,7 +1390,7 @@ int odp_pktio_promisc_mode(odp_pktio_t hdl) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", hdl); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return -1; } @@ -1421,7 +1421,7 @@ int odp_pktio_mac_addr(odp_pktio_t hdl, void *mac_addr, int addr_size) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", hdl); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return -1; } @@ -1456,7 +1456,7 @@ int odp_pktio_mac_addr_set(odp_pktio_t hdl, const void *mac_addr, int addr_size) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", hdl); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return -1; } @@ -1487,7 +1487,7 @@ odp_pktio_link_status_t odp_pktio_link_status(odp_pktio_t hdl) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", hdl); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return ODP_PKTIO_LINK_STATUS_UNKNOWN; } @@ -1543,7 +1543,7 @@ int odp_pktio_info(odp_pktio_t hdl, odp_pktio_info_t *info) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", hdl); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return -1; } @@ -1563,7 +1563,7 @@ int odp_pktio_link_info(odp_pktio_t hdl, odp_pktio_link_info_t *info) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", hdl); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return -1; } @@ -1579,7 +1579,7 @@ uint64_t odp_pktio_ts_res(odp_pktio_t hdl) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_ERR("pktio entry %d does not exist\n", hdl); + ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return 0; } @@ -1596,7 +1596,7 @@ odp_time_t odp_pktio_ts_from_ns(odp_pktio_t hdl, uint64_t ns) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_ERR("pktio entry %d does not exist\n", hdl); + ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return ODP_TIME_NULL; } @@ -1613,7 +1613,7 @@ odp_time_t odp_pktio_time(odp_pktio_t hdl, odp_time_t *global_ts) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_ERR("pktio entry %d does not exist\n", hdl); + ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return ODP_TIME_NULL; } @@ -1652,7 +1652,7 @@ void odp_pktio_print(odp_pktio_t hdl) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", hdl); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return; } @@ -1781,7 +1781,7 @@ int odp_pktio_capability(odp_pktio_t pktio, odp_pktio_capability_t *capa) entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", pktio); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -1850,7 +1850,7 @@ int odp_pktio_stats(odp_pktio_t pktio, entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", pktio); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -1880,7 +1880,7 @@ int odp_pktio_stats_reset(odp_pktio_t pktio) entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", pktio); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -1920,7 +1920,7 @@ int odp_pktin_queue_config(odp_pktio_t pktio, entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", pktio); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -2065,6 +2065,77 @@ int odp_pktin_queue_config(odp_pktio_t pktio, return 0; } +int _odp_pktio_pktout_tm_config(odp_pktio_t pktio_hdl, + odp_pktout_queue_t *queue, bool reconf) +{ + odp_pktout_queue_param_t param; + bool pktio_started = false; + odp_pktout_mode_t mode; + pktio_entry_t *entry; + unsigned int i; + int rc = 0; + + odp_pktout_queue_param_init(¶m); + param.num_queues = 1; + + entry = get_pktio_entry(pktio_hdl); + if (entry == NULL) { + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio_hdl); + return -1; + } + + rc = -ENOTSUP; + mode = entry->s.param.out_mode; + /* Don't proceed further if mode is not TM */ + if (mode != ODP_PKTOUT_MODE_TM) + return rc; + + /* Don't reconfigure unless requested */ + if (entry->s.num_out_queue && !reconf) { + *queue = entry->s.out_queue[0].pktout; + return 0; + } + + if (entry->s.state == PKTIO_STATE_STARTED) { + pktio_started = true; + rc = odp_pktio_stop(pktio_hdl); + if (rc) { + ODP_ERR("Unable to stop pktio, rc=%d\n", rc); + return rc; + } + } + + /* If re-configuring, destroy old queues */ + if (entry->s.num_out_queue) { + destroy_out_queues(entry, entry->s.num_out_queue); + entry->s.num_out_queue = 0; + } + + init_out_queues(entry); + for (i = 0; i < param.num_queues; i++) { + entry->s.out_queue[i].pktout.index = i; + entry->s.out_queue[i].pktout.pktio = pktio_hdl; + } + + entry->s.num_out_queue = param.num_queues; + + rc = 0; + if (entry->s.ops->output_queues_config) { + rc = entry->s.ops->output_queues_config(entry, ¶m); + if (rc) + ODP_ERR("Unable to setup output queues, rc=%d\n", rc); + } + + /* Return pktout queue on success */ + if (!rc) + *queue = entry->s.out_queue[0].pktout; + + /* Take pktio back to its previous state */ + if (pktio_started) + rc |= odp_pktio_start(pktio_hdl); + return rc; +} + int odp_pktout_queue_config(odp_pktio_t pktio, const odp_pktout_queue_param_t *param) { @@ -2083,7 +2154,7 @@ int odp_pktout_queue_config(odp_pktio_t pktio, entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", pktio); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -2189,7 +2260,7 @@ int odp_pktin_event_queue(odp_pktio_t pktio, odp_queue_t queues[], int num) entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", pktio); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -2229,7 +2300,7 @@ int odp_pktin_queue(odp_pktio_t pktio, odp_pktin_queue_t queues[], int num) entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", pktio); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -2268,7 +2339,7 @@ int odp_pktout_event_queue(odp_pktio_t pktio, odp_queue_t queues[], int num) entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", pktio); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -2299,7 +2370,7 @@ int odp_pktout_queue(odp_pktio_t pktio, odp_pktout_queue_t queues[], int num) entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", pktio); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -2336,7 +2407,7 @@ int odp_pktin_recv(odp_pktin_queue_t queue, odp_packet_t packets[], int num) entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", pktio); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -2365,7 +2436,7 @@ int odp_pktin_recv_tmo(odp_pktin_queue_t queue, odp_packet_t packets[], int num, entry = get_pktio_entry(queue.pktio); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", queue.pktio); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)queue.pktio); return -1; } @@ -2520,7 +2591,7 @@ int odp_pktout_send(odp_pktout_queue_t queue, const odp_packet_t packets[], entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", pktio); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -2546,7 +2617,7 @@ int odp_pktout_ts_read(odp_pktio_t hdl, odp_time_t *ts) entry = get_pktio_entry(hdl); if (odp_unlikely(entry == NULL)) { - ODP_ERR("pktio entry %d does not exist\n", hdl); + ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return -1; } diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 8dc10467a..428022ec8 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -180,7 +180,7 @@ static int queue_init_global(void) queue_capa(&capa, 0); ODP_DBG("... done.\n"); - ODP_DBG(" queue_entry_t size %u\n", sizeof(queue_entry_t)); + ODP_DBG(" queue_entry_t size %zu\n", sizeof(queue_entry_t)); ODP_DBG(" max num queues %u\n", capa.max_queues); ODP_DBG(" max queue size %u\n", capa.plain.max_size); ODP_DBG(" max num lockfree %u\n", capa.plain.lockfree.max_num); @@ -740,7 +740,7 @@ static void queue_print(odp_queue_t handle) if (!odp_pktio_info(queue->s.pktout.pktio, &pktio_info)) ODP_PRINT(" pktout %s\n", pktio_info.name); } - ODP_PRINT(" timers %" PRIu32 "\n", + ODP_PRINT(" timers %" PRIu64 "\n", odp_atomic_load_u64(&queue->s.num_timers)); ODP_PRINT(" status %s\n", queue->s.status == QUEUE_STATUS_READY ? "ready" : diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index 248855be4..ad7807bf9 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -1000,7 +1000,7 @@ static void queue_print(odp_queue_t handle) if (!odp_pktio_info(queue->s.pktout.pktio, &pktio_info)) ODP_PRINT(" pktout %s\n", pktio_info.name); } - ODP_PRINT(" timers %" PRIu32 "\n", + ODP_PRINT(" timers %" PRIu64 "\n", odp_atomic_load_u64(&queue->s.num_timers)); ODP_PRINT(" param.size %" PRIu32 "\n", queue->s.param.size); ODP_PRINT("\n"); diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index 9bef6fd7f..27c591a9c 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -91,7 +91,7 @@ static uint64_t default_huge_page_size(void) while (fgets(str, sizeof(str), file) != NULL) { if (sscanf(str, "Hugepagesize: %8lu kB", &sz) == 1) { - ODP_DBG("defaut hp size is %" PRIu64 " kB\n", sz); + ODP_DBG("defaut hp size is %lu kB\n", sz); fclose(file); return (uint64_t)sz * 1024; } diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index 2db5bc7b1..f75fbd522 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2020, Nokia + * Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -244,7 +244,7 @@ static inline timer_pool_t *handle_to_tp(odp_timer_t hdl) if (odp_likely(tp != NULL)) return timer_global->timer_pool[tp_idx]; } - ODP_ABORT("Invalid timer handle %#x\n", hdl); + ODP_ABORT("Invalid timer handle %p\n", hdl); } static inline uint32_t handle_to_idx(odp_timer_t hdl, @@ -255,7 +255,7 @@ static inline uint32_t handle_to_idx(odp_timer_t hdl, __builtin_prefetch(&tp->tick_buf[idx], 0, 0); if (odp_likely(idx < odp_atomic_load_u32(&tp->high_wm))) return idx; - ODP_ABORT("Invalid timer handle %#x\n", hdl); + ODP_ABORT("Invalid timer handle %p\n", hdl); } static inline odp_timer_t tp_idx_to_handle(timer_pool_t *tp, @@ -399,7 +399,7 @@ static odp_timer_pool_t timer_pool_new(const char *name, odp_ticketlock_unlock(&timer_global->lock); if (!odp_global_rw->inline_timers) { - if (tp->param.clk_src == ODP_CLOCK_CPU) + if (tp->param.clk_src == ODP_CLOCK_DEFAULT) itimer_init(tp); } else { /* Update the highest index for inline timer scan */ @@ -441,7 +441,7 @@ static void odp_timer_pool_del(timer_pool_t *tp) if (!odp_global_rw->inline_timers) { /* Stop POSIX itimer signals */ - if (tp->param.clk_src == ODP_CLOCK_CPU) + if (tp->param.clk_src == ODP_CLOCK_DEFAULT) itimer_fini(tp); stop_timer_thread(tp); @@ -1220,8 +1220,8 @@ static void itimer_fini(timer_pool_t *tp) int odp_timer_capability(odp_timer_clk_src_t clk_src, odp_timer_capability_t *capa) { - if (clk_src != ODP_CLOCK_CPU) { - ODP_ERR("Only CPU clock source supported\n"); + if (clk_src != ODP_CLOCK_DEFAULT) { + ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", clk_src); return -1; } @@ -1248,8 +1248,8 @@ int odp_timer_capability(odp_timer_clk_src_t clk_src, int odp_timer_res_capability(odp_timer_clk_src_t clk_src, odp_timer_res_capability_t *res_capa) { - if (clk_src != ODP_CLOCK_CPU) { - ODP_ERR("Only CPU clock source supported\n"); + if (clk_src != ODP_CLOCK_DEFAULT) { + ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", clk_src); return -1; } @@ -1385,13 +1385,13 @@ int odp_timer_set_abs(odp_timer_t hdl, uint32_t idx = handle_to_idx(hdl, tp); if (odp_unlikely(abs_tck < cur_tick + tp->min_rel_tck)) - return ODP_TIMER_TOOEARLY; + return ODP_TIMER_TOO_NEAR; if (odp_unlikely(abs_tck > cur_tick + tp->max_rel_tck)) - return ODP_TIMER_TOOLATE; + return ODP_TIMER_TOO_FAR; if (timer_reset(idx, abs_tck, (odp_buffer_t *)tmo_ev, tp)) return ODP_TIMER_SUCCESS; else - return ODP_TIMER_NOEVENT; + return ODP_TIMER_FAIL; } int odp_timer_set_rel(odp_timer_t hdl, @@ -1404,13 +1404,13 @@ int odp_timer_set_rel(odp_timer_t hdl, uint32_t idx = handle_to_idx(hdl, tp); if (odp_unlikely(rel_tck < tp->min_rel_tck)) - return ODP_TIMER_TOOEARLY; + return ODP_TIMER_TOO_NEAR; if (odp_unlikely(rel_tck > tp->max_rel_tck)) - return ODP_TIMER_TOOLATE; + return ODP_TIMER_TOO_FAR; if (timer_reset(idx, abs_tck, (odp_buffer_t *)tmo_ev, tp)) return ODP_TIMER_SUCCESS; else - return ODP_TIMER_NOEVENT; + return ODP_TIMER_FAIL; } int odp_timer_cancel(odp_timer_t hdl, odp_event_t *tmo_ev) diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index 1633bc454..cd7a9ecd9 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -22,6 +22,7 @@ #include <odp/api/std_types.h> #include <protocols/eth.h> #include <protocols/ip.h> +#include <odp_packet_io_internal.h> #include <odp_traffic_mngr_internal.h> #include <odp/api/plat/packet_inlines.h> #include <odp/api/plat/byteorder_inlines.h> @@ -610,8 +611,8 @@ static void tm_shaper_params_cvt_to(const odp_tm_shaper_params_t *shaper_params, uint32_t min_time_delta; int64_t commit_burst, peak_burst; - commit_rate = tm_bps_to_rate(shaper_params->commit_bps); - if ((shaper_params->commit_bps == 0) || (commit_rate == 0)) { + commit_rate = tm_bps_to_rate(shaper_params->commit_rate); + if ((shaper_params->commit_rate == 0) || (commit_rate == 0)) { tm_shaper_params->max_commit_time_delta = 0; tm_shaper_params->max_peak_time_delta = 0; tm_shaper_params->commit_rate = 0; @@ -628,8 +629,8 @@ static void tm_shaper_params_cvt_to(const odp_tm_shaper_params_t *shaper_params, max_commit_time_delta = tm_max_time_delta(commit_rate); commit_burst = (int64_t)shaper_params->commit_burst; - peak_rate = tm_bps_to_rate(shaper_params->peak_bps); - if ((shaper_params->peak_bps == 0) || (peak_rate == 0)) { + peak_rate = tm_bps_to_rate(shaper_params->peak_rate); + if ((shaper_params->peak_rate == 0) || (peak_rate == 0)) { peak_rate = 0; max_peak_time_delta = 0; peak_burst = 0; @@ -669,8 +670,8 @@ static void tm_shaper_params_cvt_from(tm_shaper_params_t *tm_shaper_params, commit_burst = tm_shaper_params->max_commit >> (26 - 3); peak_burst = tm_shaper_params->max_peak >> (26 - 3); - odp_shaper_params->commit_bps = commit_bps; - odp_shaper_params->peak_bps = peak_bps; + odp_shaper_params->commit_rate = commit_bps; + odp_shaper_params->peak_rate = peak_bps; odp_shaper_params->commit_burst = (uint32_t)commit_burst; odp_shaper_params->peak_burst = (uint32_t)peak_burst; odp_shaper_params->shaper_len_adjust = tm_shaper_params->len_adjust; @@ -2565,6 +2566,12 @@ int odp_tm_capabilities(odp_tm_capabilities_t capabilities[] ODP_UNUSED, cap_ptr->ecn_marking_supported = true; cap_ptr->drop_prec_marking_supported = true; + cap_ptr->dynamic_topology_update = true; + cap_ptr->dynamic_shaper_update = true; + cap_ptr->dynamic_sched_update = true; + cap_ptr->dynamic_wred_update = true; + cap_ptr->dynamic_threshold_update = true; + for (color = 0; color < ODP_NUM_PACKET_COLORS; color++) cap_ptr->marking_colors_supported[color] = true; @@ -2618,6 +2625,12 @@ static void tm_system_capabilities_set(odp_tm_capabilities_t *cap_ptr, cap_ptr->drop_prec_marking_supported = req_ptr->drop_prec_marking_needed; + cap_ptr->dynamic_topology_update = true; + cap_ptr->dynamic_shaper_update = true; + cap_ptr->dynamic_sched_update = true; + cap_ptr->dynamic_wred_update = true; + cap_ptr->dynamic_threshold_update = true; + for (color = 0; color < ODP_NUM_PACKET_COLORS; color++) cap_ptr->marking_colors_supported[color] = req_ptr->marking_colors_needed[color]; @@ -2919,15 +2932,20 @@ odp_tm_t odp_tm_create(const char *name, return ODP_TM_INVALID; } + odp_ticketlock_lock(&tm_glb->create_lock); + /* If we are using pktio output (usual case) get the first associated * pktout_queue for this pktio and fail if there isn't one. */ - if (egress->egress_kind == ODP_TM_EGRESS_PKT_IO && - odp_pktout_queue(egress->pktio, &pktout, 1) != 1) - return ODP_TM_INVALID; + if (egress->egress_kind == ODP_TM_EGRESS_PKT_IO) { + rc = _odp_pktio_pktout_tm_config(egress->pktio, &pktout, false); + if (rc) { + odp_ticketlock_unlock(&tm_glb->create_lock); + return ODP_TM_INVALID; + } + } /* Allocate tm_system_t record. */ - odp_ticketlock_lock(&tm_glb->create_lock); tm_system = tm_system_alloc(); if (!tm_system) { odp_ticketlock_unlock(&tm_glb->create_lock); @@ -3042,6 +3060,22 @@ int odp_tm_capability(odp_tm_t odp_tm, odp_tm_capabilities_t *capabilities) return 0; } +int odp_tm_start(odp_tm_t odp_tm) +{ + (void)odp_tm; + + /* Nothing more to do after TM create */ + return 0; +} + +int odp_tm_stop(odp_tm_t odp_tm) +{ + (void)odp_tm; + + /* Nothing more to do for topology changes */ + return 0; +} + int odp_tm_destroy(odp_tm_t odp_tm) { tm_system_t *tm_system; @@ -3065,6 +3099,7 @@ int odp_tm_destroy(odp_tm_t odp_tm) _odp_queue_pool_destroy(tm_system->_odp_int_queue_pool); _odp_timer_wheel_destroy(tm_system->_odp_int_timer_wheel); + _odp_int_name_tbl_delete(tm_system->name_tbl_id); tm_system_free(tm_system); return 0; } @@ -3204,6 +3239,10 @@ odp_tm_shaper_t odp_tm_shaper_create(const char *name, odp_tm_shaper_t shaper_handle; _odp_int_name_t name_tbl_id; + /* We don't support shaper in packet mode */ + if (params->packet_mode) + return ODP_TM_INVALID; + profile_obj = tm_common_profile_create(name, TM_SHAPER_PROFILE, &shaper_handle, &name_tbl_id); if (!profile_obj) @@ -3573,7 +3612,7 @@ int odp_tm_wred_destroy(odp_tm_wred_t wred_profile) return -1; return tm_common_profile_destroy(wred_profile, - ODP_INVALID_NAME); + wred_params->name_tbl_id); } int odp_tm_wred_params_read(odp_tm_wred_t wred_profile, @@ -4685,10 +4724,10 @@ void odp_tm_stats_print(odp_tm_t odp_tm) ODP_PRINT("odp_tm_stats_print - tm_system=0x%" PRIX64 " tm_idx=%u\n", odp_tm, tm_system->tm_idx); - ODP_PRINT(" input_work_queue size=%u current cnt=%u peak cnt=%u\n", - INPUT_WORK_RING_SIZE, input_work_queue->queue_cnt, + ODP_PRINT(" input_work_queue size=%u current cnt=%" PRIu64 " peak cnt=%" PRIu32 "\n", + INPUT_WORK_RING_SIZE, odp_atomic_load_u64(&input_work_queue->queue_cnt), input_work_queue->peak_cnt); - ODP_PRINT(" input_work_queue enqueues=%" PRIu64 " dequeues=% " PRIu64 + ODP_PRINT(" input_work_queue enqueues=%" PRIu64 " dequeues=%" PRIu64 " fail_cnt=%" PRIu64 "\n", input_work_queue->total_enqueues, input_work_queue->total_dequeues, input_work_queue->enqueue_fail_cnt); diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 6b349293a..5d5fead16 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -1286,7 +1286,7 @@ static int dpdk_pktio_init(void) masklen = odp_cpumask_to_str(&mask, mask_str, ODP_CPUMASK_STR_SIZE); if (masklen < 0) { - ODP_ERR("CPU mask error: d\n", masklen); + ODP_ERR("CPU mask error: %" PRId32 "\n", masklen); return -1; } diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c index e0c8cba3b..ad633e231 100644 --- a/platform/linux-generic/pktio/ipc.c +++ b/platform/linux-generic/pktio/ipc.c @@ -322,7 +322,7 @@ static int _ipc_init_master(pktio_entry_t *pktio_entry, pinfo = pktio_ipc->pinfo; pool_name = _ipc_odp_buffer_pool_shm_name(pool_hdl); if (strlen(pool_name) > ODP_POOL_NAME_LEN) { - ODP_ERR("pid %d ipc pool name %s is too big %d\n", + ODP_ERR("pid %d ipc pool name %s is too big %zu\n", getpid(), pool_name, strlen(pool_name)); goto free_s_prod; } @@ -742,7 +742,7 @@ static int ipc_pktio_recv_lockless(pktio_entry_t *pktio_entry, ring_ptr_enq_multi(r_p, ring_mask, ipcbufs_p, pkts); for (i = 0; i < pkts; i++) { - IPC_ODP_DBG("%d/%d send to be free packet offset %x\n", + IPC_ODP_DBG("%d/%d send to be free packet offset %" PRIuPTR "\n", i, pkts, offsets[i]); } @@ -820,7 +820,7 @@ static int ipc_pktio_send_lockless(pktio_entry_t *pktio_entry, /* compile all function code even if ipc disabled with config */ IPC_ODP_DBG("%d/%d send packet %" PRIu64 ", pool %" PRIu64 "," - "phdr = %p, offset %x, sendoff %x, addr %p iaddr " + "phdr = %p, offset %td, sendoff %" PRIxPTR ", addr %p iaddr " "%p\n", i, num, odp_packet_to_u64(pkt), odp_pool_to_u64(pool_hdl), pkt_hdr, (uint8_t *)pkt_hdr->seg_data - diff --git a/platform/linux-generic/pktio/pktio_common.c b/platform/linux-generic/pktio/pktio_common.c index 4090f8063..d2f4d7219 100644 --- a/platform/linux-generic/pktio/pktio_common.c +++ b/platform/linux-generic/pktio/pktio_common.c @@ -7,6 +7,7 @@ #include <odp_packet_io_internal.h> #include <errno.h> +#include <inttypes.h> static int sock_recv_mq_tmo_select(pktio_entry_t * const *entry, const int index[], @@ -71,8 +72,8 @@ int _odp_sock_recv_mq_tmo_try_int_driven(const struct odp_pktin_queue_t queues[] entry[i] = get_pktio_entry(queues[i].pktio); index[i] = queues[i].index; if (entry[i] == NULL) { - ODP_DBG("pktio entry %d does not exist\n", - queues[i].pktio); + ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", + (uintptr_t)queues[i].pktio); *trial_successful = 0; return -1; } diff --git a/platform/linux-generic/test/Makefile.am b/platform/linux-generic/test/Makefile.am index 1f7ae611e..049ce2187 100644 --- a/platform/linux-generic/test/Makefile.am +++ b/platform/linux-generic/test/Makefile.am @@ -8,25 +8,13 @@ TESTS_ENVIRONMENT += WITH_OPENSSL=0 endif SUBDIRS = - -if WITH_EXAMPLES -TESTS = ipsec/ipsec_api_example.sh \ - ipsec/ipsec_crypto_example.sh - -dist_check_SCRIPTS = ipsec/ipsec_api_example.sh \ - ipsec/ipsec_crypto_example.sh -else TESTS = -dist_check_SCRIPTS = -endif if test_vald TESTS += validation/api/pktio/pktio_run.sh \ validation/api/pktio/pktio_run_tap.sh \ validation/api/shmem/shmem_linux$(EXEEXT) -test_SCRIPTS = $(dist_check_SCRIPTS) - SUBDIRS += validation/api/pktio\ validation/api/shmem\ pktio_ipc \ @@ -69,25 +57,3 @@ if test_installdir installcheck-local: $(DESTDIR)/$(testdir)/run-test.sh $(TESTNAME) endif - -# If building out-of-tree, make check will not copy the scripts and data to the -# $(builddir) assuming that all commands are run locally. However this prevents -# running tests on a remote target using LOG_COMPILER. -# So copy all script and data files explicitly here. -all-local: - if [ "x$(srcdir)" != "x$(builddir)" ]; then \ - for f in $(dist_check_SCRIPTS); do \ - if [ -e $(srcdir)/$$f ]; then \ - mkdir -p $(builddir)/$$(dirname $$f); \ - cp -f $(srcdir)/$$f $(builddir)/$$f; \ - fi \ - done \ - fi -clean-local: - if [ "x$(srcdir)" != "x$(builddir)" ]; then \ - for f in $(dist_check_SCRIPTS); do \ - rm -f $(builddir)/$$f; \ - done \ - fi - -.NOTPARALLEL: diff --git a/platform/linux-generic/test/example/Makefile.am b/platform/linux-generic/test/example/Makefile.am index 22b254cd7..947647cd4 100644 --- a/platform/linux-generic/test/example/Makefile.am +++ b/platform/linux-generic/test/example/Makefile.am @@ -1,6 +1,8 @@ SUBDIRS = \ classifier \ generator \ + ipsec_api \ + ipsec_crypto \ l2fwd_simple \ l3fwd \ packet \ diff --git a/platform/linux-generic/test/example/ipsec_api/Makefile.am b/platform/linux-generic/test/example/ipsec_api/Makefile.am new file mode 100644 index 000000000..101c97cdf --- /dev/null +++ b/platform/linux-generic/test/example/ipsec_api/Makefile.am @@ -0,0 +1,23 @@ +EXTRA_DIST = pktio_env + +# If building out-of-tree, make check will not copy the scripts and data to the +# $(builddir) assuming that all commands are run locally. However this prevents +# running tests on a remote target using LOG_COMPILER. +# So copy all script and data files explicitly here. +all-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + if [ -e $(srcdir)/$$f ]; then \ + mkdir -p $(builddir)/$$(dirname $$f); \ + cp -f $(srcdir)/$$f $(builddir)/$$f; \ + fi \ + done \ + fi +clean-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + rm -f $(builddir)/$$f; \ + done \ + fi + +.NOTPARALLEL: diff --git a/platform/linux-generic/test/example/ipsec_api/pktio_env b/platform/linux-generic/test/example/ipsec_api/pktio_env new file mode 100644 index 000000000..a16002326 --- /dev/null +++ b/platform/linux-generic/test/example/ipsec_api/pktio_env @@ -0,0 +1,66 @@ +#!/bin/sh +# +# Copyright (C) 2021, Marvell +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# +# Script to setup interfaces used for running application on linux-generic. +# +# ipsec_api application uses two loop devices loop0 and loop1. +# + +if [ "$0" == "$BASH_SOURCE" ]; then + echo "Error: Platform specific env file has to be sourced." + exit 1 +fi + +# Absolute path to the .env file. +LINUX_ENV_PATH=$PWD/../../platform/linux-generic/test + +TESTENV="tests-linux-generic.env" + +if [ -f $LINUX_ENV_PATH/$TESTENV ]; then + source $LINUX_ENV_PATH/$TESTENV +else + echo "BUG: unable to find $TESTENV!" + echo "$TESTENV has to be in following directory: " + echo " $LINUX_ENV_PATH" + exit 1 +fi + +# Skip IPsec example tests when there's no OpenSSL. +if [ -n "$WITH_OPENSSL" ] && [ ${WITH_OPENSSL} -eq 0 ]; then + echo "Crypto not supported. Skipping." + exit 77 +fi + +IF0=p7p1 +IF1=p8p1 + +NEXT_HOP_MAC0=08:00:27:76:B5:E0 +NEXT_HOP_MAC1=08:00:27:F5:8B:DB + +LIF0=loop1 +LIF1=loop2 + +IF_LIST=$LIF0,$LIF1 +ROUTE_IF_INB=$LIF0 +ROUTE_IF_OUTB=$LIF1 +OUT_IF=$LIF1 +IN_IF=$LIF0 + +validate_result() +{ + return 0 +} + +setup_interfaces() +{ + return 0 +} + +cleanup_interfaces() +{ + return 0 +} diff --git a/platform/linux-generic/test/example/ipsec_crypto/Makefile.am b/platform/linux-generic/test/example/ipsec_crypto/Makefile.am new file mode 100644 index 000000000..101c97cdf --- /dev/null +++ b/platform/linux-generic/test/example/ipsec_crypto/Makefile.am @@ -0,0 +1,23 @@ +EXTRA_DIST = pktio_env + +# If building out-of-tree, make check will not copy the scripts and data to the +# $(builddir) assuming that all commands are run locally. However this prevents +# running tests on a remote target using LOG_COMPILER. +# So copy all script and data files explicitly here. +all-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + if [ -e $(srcdir)/$$f ]; then \ + mkdir -p $(builddir)/$$(dirname $$f); \ + cp -f $(srcdir)/$$f $(builddir)/$$f; \ + fi \ + done \ + fi +clean-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + rm -f $(builddir)/$$f; \ + done \ + fi + +.NOTPARALLEL: diff --git a/platform/linux-generic/test/example/ipsec_crypto/pktio_env b/platform/linux-generic/test/example/ipsec_crypto/pktio_env new file mode 100644 index 000000000..99a1ac42b --- /dev/null +++ b/platform/linux-generic/test/example/ipsec_crypto/pktio_env @@ -0,0 +1,72 @@ +#!/bin/sh +# +# Copyright (C) 2021, Marvell +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# +# Script to setup interfaces used for running application on linux-generic. +# +# ipsec_api application uses two loop devices loop0 and loop1. +# + +if [ "$0" == "$BASH_SOURCE" ]; then + echo "Error: Platform specific env file has to be sourced." + exit 1 +fi + +# Absolute path to the .env file. +LINUX_ENV_PATH=$PWD/../../platform/linux-generic/test + +TESTENV="tests-linux-generic.env" + +if [ -f $LINUX_ENV_PATH/$TESTENV ]; then + source $LINUX_ENV_PATH/$TESTENV +else + echo "BUG: unable to find $TESTENV!" + echo "$TESTENV has to be in following directory: " + echo " $LINUX_ENV_PATH" + exit 1 +fi + +# Skip IPsec example tests when there's no OpenSSL. +if [ -n "$WITH_OPENSSL" ] && [ ${WITH_OPENSSL} -eq 0 ]; then + echo "Crypto not supported. Skipping." + exit 77 +fi + +# Skip live and router mode tests. +if [ ${IPSEC_APP_MODE} -eq 1 ] || [ ${IPSEC_APP_MODE} -eq 2 ]; then + echo "Live / Router mode test. Skipping." + exit 77 +fi + +IF0=p7p1 +IF1=p8p1 + +NEXT_HOP_MAC0=08:00:27:76:B5:E0 +NEXT_HOP_MAC1=08:00:27:F5:8B:DB + +LIF0=loop1 +LIF1=loop2 + +IF_LIST=$LIF0,$LIF1 +ROUTE_IF_INB=$LIF0 +ROUTE_IF_OUTB=$LIF1 +OUT_IF=$LIF1 +IN_IF=$LIF0 + +validate_result() +{ + return 0 +} + +setup_interfaces() +{ + return 0 +} + +cleanup_interfaces() +{ + return 0 +} diff --git a/platform/linux-generic/test/inline-timer.conf b/platform/linux-generic/test/inline-timer.conf index e4d4af307..93195f5a8 100644 --- a/platform/linux-generic/test/inline-timer.conf +++ b/platform/linux-generic/test/inline-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.15" +config_file_version = "0.1.16" timer: { # Enable inline timer implementation diff --git a/platform/linux-generic/test/ipsec/ipsec_api_example.sh b/platform/linux-generic/test/ipsec/ipsec_api_example.sh deleted file mode 100755 index 82fd193d9..000000000 --- a/platform/linux-generic/test/ipsec/ipsec_api_example.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2019, Nokia -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause -# - -# Skip IPsec example tests when there's no OpenSSL. -if [ -n "$WITH_OPENSSL" ] && [ ${WITH_OPENSSL} -eq 0 ]; then -echo "Crypto not supported. Skipping." -exit 77 -fi - -# Absolute path to the example binary. This is needed during distcheck, which -# keeps scripts and binaries in different directories (scripts are not copied -# into the distribution directory). -export IPSEC_EXAMPLE_PATH=$(pwd)/../../../example/ipsec_api - -declare -i RESULT=0 - -pushd $(dirname $0)/../../../../example/ipsec_api - -./odp_ipsec_api_run_simple.sh -RESULT+=$? - -./odp_ipsec_api_run_esp_out.sh -RESULT+=$? - -popd - -exit ${RESULT} diff --git a/platform/linux-generic/test/ipsec/ipsec_crypto_example.sh b/platform/linux-generic/test/ipsec/ipsec_crypto_example.sh deleted file mode 100755 index d99fc3bd2..000000000 --- a/platform/linux-generic/test/ipsec/ipsec_crypto_example.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2019, Nokia -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause -# - -# Skip IPsec example tests when there's no OpenSSL. -if [ -n "$WITH_OPENSSL" ] && [ ${WITH_OPENSSL} -eq 0 ]; then -echo "Crypto not supported. Skipping." -exit 77 -fi - -# Absolute path to the example binary. This is needed during distcheck, which -# keeps scripts and binaries in different directories (scripts are not copied -# into the distribution directory). -export IPSEC_EXAMPLE_PATH=$(pwd)/../../../example/ipsec_crypto - -declare -i RESULT=0 - -pushd $(dirname $0)/../../../../example/ipsec_crypto - -./odp_ipsec_crypto_run_simple.sh -RESULT+=$? - -./odp_ipsec_crypto_run_esp_out.sh -RESULT+=$? - -popd - -exit ${RESULT} diff --git a/platform/linux-generic/test/packet_align.conf b/platform/linux-generic/test/packet_align.conf index 9b37752ba..58a73f2df 100644 --- a/platform/linux-generic/test/packet_align.conf +++ b/platform/linux-generic/test/packet_align.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.15" +config_file_version = "0.1.16" pool: { pkt: { diff --git a/platform/linux-generic/test/process-mode.conf b/platform/linux-generic/test/process-mode.conf index 5354cae2f..a6e6080d2 100644 --- a/platform/linux-generic/test/process-mode.conf +++ b/platform/linux-generic/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.15" +config_file_version = "0.1.16" # Shared memory options shm: { diff --git a/platform/linux-generic/test/sched-basic.conf b/platform/linux-generic/test/sched-basic.conf index 57a8a772c..79537b454 100644 --- a/platform/linux-generic/test/sched-basic.conf +++ b/platform/linux-generic/test/sched-basic.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.15" +config_file_version = "0.1.16" sched_basic: { # Test scheduler with an odd spread value |