diff options
Diffstat (limited to 'platform')
46 files changed, 1631 insertions, 562 deletions
diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 0ce6ccc03..5e9f40de5 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -98,7 +98,6 @@ noinst_HEADERS = \ ${top_srcdir}/platform/linux-generic/include/odp_libconfig_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_llqueue.h \ ${top_srcdir}/platform/linux-generic/include/odp_macros_internal.h \ - include/odp_packet_dpdk.h \ include/odp_packet_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_name_table_internal.h \ include/odp_packet_io_internal.h \ @@ -152,7 +151,7 @@ __LIB__libodp_linux_la_SOURCES = \ ../linux-generic/odp_ipsec_events.c \ ../linux-generic/odp_ipsec_sad.c \ ../linux-generic/odp_name_table.c \ - odp_libconfig.c \ + ../linux-generic/odp_libconfig.c \ odp_packet.c \ odp_packet_dpdk.c \ odp_packet_flags.c \ diff --git a/platform/linux-dpdk/README b/platform/linux-dpdk/README index 71cced161..c9e7af78e 100644 --- a/platform/linux-dpdk/README +++ b/platform/linux-dpdk/README @@ -4,13 +4,6 @@ All rights reserved. SPDX-License-Identifier: BSD-3-Clause -ERRATA: -- DPDK 16.07 and earlier supports pool names with RTE_MEMZONE_NAMESIZE - characters (including terminating NULL), which is 6 characters less than - ODP_POOL_NAME_LEN. Names reaching into this interval might collide if the - first 25 characters are not unique. - - 1. Rationale ================================================= @@ -180,16 +173,15 @@ To restore the NIC's back to kernel use something like this: 5. Running ODP apps ================================================= -ODP-DPDK applications need to be run as root. You also need to supply the DPDK -command line parameters either as a null-terminated array of char's to -odp_global_init()'s platform_params parameter: +ODP-DPDK applications need to be run as root. You may also need to +supply DPDK command line parameters either as a null-terminated array of +char's to odp_global_init()'s platform_params parameter: - odp_global_init([params], "-n 4"); + odp_global_init([params], "--no-huge"); Or, if it's NULL the platform tries to read the ODP_PLATFORM_PARAMS environment variable. -You need to pass at least "-n [1..4]" to specify the number of memory channels. The coremask (-c) is calculated by ODP-DPDK based on the process affinity at startup. You can influence that with 'taskset'. DPDK init changes the affinity of the calling thread, so after it returns the original affinity is restored. @@ -201,7 +193,7 @@ like proper DPDK threads. Exaple how to run an ODP-DPDK L2 forwarding application: - sudo ODP_PLATFORM_PARAMS="-n 4" ./odp_l2fwd -i 0,1 -c 2 + sudo ./odp_l2fwd -i 0,1 -c 2 -i 0,1 - interface numbers -c 2 - number of worker cpus @@ -234,7 +226,7 @@ CONFIG_RTE_LIBRTE_PMD_PCAP=y mount -t hugetlbfs none /mnt/huge Finally give l2fwd fake devices: - ./l2fwd -c '0xf' -n 4 --vdev "eth_pcap0,iface=veth2-1" --vdev="eth_pcap1,iface=veth2-3" -- -p 3 + ./l2fwd -c '0xf' --vdev "eth_pcap0,iface=veth2-1" --vdev="eth_pcap1,iface=veth2-3" -- -p 3 7. Upgrading ODP-DPDK to newer ODP API level ================================================= @@ -270,8 +262,8 @@ http://dpdk.org/doc/guides/cryptodevs/index.html. To build odp-dpdk with dpdk virtual crypto devices, we need to build supporting intel multi-buffer library prior to dpdk build. -get the Intel multi-buffer crypto from, -https://downloadcenter.intel.com/download/22972 +Get the Intel multi-buffer crypto library from, +https://github.com/intel/intel-ipsec-mb and follow the README from the repo on how to build the library. building dpdk: @@ -294,5 +286,5 @@ AESNI_MULTI_BUFFER_LIB_PATH=/path-to/Intel-multi-buffer-crypto/ \ when building odp-dpdk application, add the multi-buffer crypto library path to make file. Before running the application, export ODP_PLATFORM_PARAMS with corresponding crypto vdev's. -ex: ODP_PLATFORM_PARAMS="-n 4 --vdev cryptodev_aesni_mb_pmd,max_nb_sessions=32 \ +ex: ODP_PLATFORM_PARAMS="--vdev cryptodev_aesni_mb_pmd,max_nb_sessions=32 \ --vdev cryptodev_null_pmd,max_nb_sessions=32" diff --git a/platform/linux-dpdk/include/odp/api/plat/std_clib_inlines.h b/platform/linux-dpdk/include/odp/api/plat/std_clib_inlines.h index cf556e042..14072ba52 100644 --- a/platform/linux-dpdk/include/odp/api/plat/std_clib_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/std_clib_inlines.h @@ -12,6 +12,8 @@ extern "C" { #endif #include <string.h> + +#include <rte_config.h> #include <rte_memcpy.h> _ODP_INLINE void *odp_memcpy(void *dst, const void *src, size_t num) diff --git a/platform/linux-dpdk/include/odp_packet_dpdk.h b/platform/linux-dpdk/include/odp_packet_dpdk.h deleted file mode 100644 index a2c9132b9..000000000 --- a/platform/linux-dpdk/include/odp_packet_dpdk.h +++ /dev/null @@ -1,50 +0,0 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_PACKET_DPDK_H -#define ODP_PACKET_DPDK_H - -#include <stdint.h> -#include <net/if.h> - -#include <protocols/eth.h> -#include <odp/api/align.h> -#include <odp/api/debug.h> -#include <odp/api/packet.h> -#include <odp_packet_internal.h> -#include <odp/api/pool.h> -#include <odp_pool_internal.h> -#include <odp_buffer_internal.h> -#include <odp/api/std_types.h> - -#include <rte_config.h> -#include <rte_memory.h> -#include <rte_memzone.h> -#include <rte_launch.h> -#include <rte_tailq.h> -#include <rte_eal.h> -#include <rte_per_lcore.h> -#include <rte_lcore.h> -#include <rte_branch_prediction.h> -#include <rte_prefetch.h> -#include <rte_cycles.h> -#include <rte_errno.h> -#include <rte_debug.h> -#include <rte_log.h> -#include <rte_byteorder.h> -#include <rte_pci.h> -#include <rte_random.h> -#include <rte_ether.h> -#include <rte_ethdev.h> -#include <rte_ip.h> -#include <rte_ip_frag.h> -#include <rte_udp.h> -#include <rte_tcp.h> -#include <rte_hash.h> -#include <rte_jhash.h> -#include <rte_hash_crc.h> - -#endif diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h index 328b499ff..2268833f8 100644 --- a/platform/linux-dpdk/include/odp_packet_internal.h +++ b/platform/linux-dpdk/include/odp_packet_internal.h @@ -142,6 +142,11 @@ static inline odp_packet_hdr_t *packet_hdr(odp_packet_t pkt) return (odp_packet_hdr_t *)(uintptr_t)pkt; } +static inline odp_packet_t packet_handle(odp_packet_hdr_t *pkt_hdr) +{ + return (odp_packet_t)pkt_hdr; +} + static inline struct rte_mbuf *pkt_to_mbuf(odp_packet_t pkt) { return (struct rte_mbuf *)(uintptr_t)pkt; @@ -256,6 +261,10 @@ int _odp_packet_set_data(odp_packet_t pkt, uint32_t offset, int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset, const void *s, uint32_t len); +int _odp_packet_ipv4_chksum_insert(odp_packet_t pkt); +int _odp_packet_tcp_chksum_insert(odp_packet_t pkt); +int _odp_packet_udp_chksum_insert(odp_packet_t pkt); + /* We can't enforce tailroom reservation for received packets */ ODP_STATIC_ASSERT(CONFIG_PACKET_TAILROOM == 0, "ERROR: Tailroom has to be 0, DPDK doesn't support this"); diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h index 963ea7ff9..6160cfb66 100644 --- a/platform/linux-dpdk/include/odp_packet_io_internal.h +++ b/platform/linux-dpdk/include/odp_packet_io_internal.h @@ -31,7 +31,6 @@ extern "C" { #define PKTIO_MAX_QUEUES 64 #include <linux/if_ether.h> -#include <odp_packet_dpdk.h> #include <odp_packet_null.h> #define PKTIO_NAME_LEN 256 diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 6b323b5e2..8be6bc146 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -21,6 +21,8 @@ #include <odp/api/packet.h> #include <odp/api/plat/packet_inlines.h> #include <odp_packet_internal.h> + +#include <rte_config.h> #include <rte_crypto.h> #include <rte_cryptodev.h> diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c index 7a51ff23b..2729d6a6f 100644 --- a/platform/linux-dpdk/odp_init.c +++ b/platform/linux-dpdk/odp_init.c @@ -7,7 +7,6 @@ #include "config.h" #include <odp_posix_extensions.h> -#include <odp_packet_dpdk.h> #include <odp/api/init.h> #include <odp_debug_internal.h> #include <odp/api/debug.h> @@ -25,6 +24,11 @@ #include <errno.h> #include <sys/types.h> #include <pwd.h> +#include <ctype.h> + +#include <rte_config.h> +#include <rte_eal.h> +#include <rte_string_fns.h> #define MEMPOOL_OPS(hdl) extern void mp_hdlr_init_##hdl(void) MEMPOOL_OPS(ops_mp_mc); @@ -50,23 +54,6 @@ void refer_constructors(void) } #endif -static void print_dpdk_env_help(void) -{ - char prgname[] = "odpdpdk"; - char help_str[] = "--help"; - char *dpdk_argv[] = {prgname, help_str}; - int dpdk_argc = 2; - - ODP_ERR("Neither (char *)platform_params were provided to " - "odp_init_global(),\n"); - ODP_ERR("nor ODP_PLATFORM_PARAMS environment variable were " - "specified.\n"); - ODP_ERR("A string of DPDK command line arguments should be provided"); - ODP_ERR("Example: export ODP_PLATFORM_PARAMS=\"-n 4 --no-huge\"\n"); - ODP_ERR("Note: -c argument substitutes automatically from odp coremask\n"); - rte_eal_init(dpdk_argc, dpdk_argv); -} - static int odp_init_dpdk(const char *cmdline) { char **dpdk_argv; @@ -80,10 +67,8 @@ static int odp_init_dpdk(const char *cmdline) if (cmdline == NULL) { cmdline = getenv("ODP_PLATFORM_PARAMS"); - if (cmdline == NULL) { - print_dpdk_env_help(); - return -1; - } + if (cmdline == NULL) + cmdline = ""; } CPU_ZERO(&original_cpuset); diff --git a/platform/linux-dpdk/odp_libconfig.c b/platform/linux-dpdk/odp_libconfig.c deleted file mode 100644 index 6d5ee524e..000000000 --- a/platform/linux-dpdk/odp_libconfig.c +++ /dev/null @@ -1,99 +0,0 @@ -/* Copyright (c) 2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include "config.h" - -#include <stdlib.h> -#include <string.h> -#include <libconfig.h> - -#include <odp/api/version.h> -#include <odp_internal.h> -#include <odp_debug_internal.h> -#include <odp_libconfig_internal.h> -#include <odp_libconfig_config.h> - -#define CONF_STR_NAME ((const char *)odp_linux_dpdk_conf) - -extern struct odp_global_data_s odp_global_data; - -int _odp_libconfig_init_global(void) -{ - const char *filename; - const char *vers; - const char *vers_rt; - const char *ipml; - const char *ipml_rt; - config_t *config = &odp_global_data.libconfig_default; - config_t *config_rt = &odp_global_data.libconfig_runtime; - - config_init(config); - config_init(config_rt); - - if (!config_read_string(config, CONF_STR_NAME)) { - ODP_ERR("Failed to read default config: %s(%d): %s\n", - config_error_file(config), config_error_line(config), - config_error_text(config)); - goto fail; - } - - filename = getenv("ODP_CONFIG_FILE"); - if (filename == NULL) - return 0; - - if (!config_read_file(config_rt, filename)) { - ODP_ERR("Failed to read config file: %s(%d): %s\n", - config_error_file(config_rt), - config_error_line(config_rt), - config_error_text(config_rt)); - goto fail; - } - - /* Check runtime configuration's implementation name and version */ - if (!config_lookup_string(config, "odp_implementation", &ipml) || - !config_lookup_string(config_rt, "odp_implementation", &ipml_rt)) { - ODP_ERR("Configuration missing 'odp_implementation' field\n"); - goto fail; - } - if (!config_lookup_string(config, "config_file_version", &vers) || - !config_lookup_string(config_rt, "config_file_version", &vers_rt)) { - ODP_ERR("Configuration missing 'config_file_version' field\n"); - goto fail; - } - if (strcmp(vers, vers_rt) || strcmp(ipml, ipml_rt)) { - ODP_ERR("Runtime configuration mismatch\n"); - goto fail; - } - - return 0; -fail: - config_destroy(config); - config_destroy(config_rt); - return -1; -} - -int _odp_libconfig_term_global(void) -{ - config_destroy(&odp_global_data.libconfig_default); - config_destroy(&odp_global_data.libconfig_runtime); - - return 0; -} - -int _odp_libconfig_lookup_int(const char *path, int *value) -{ - int ret_def = CONFIG_FALSE; - int ret_rt = CONFIG_FALSE; - - ret_def = config_lookup_int(&odp_global_data.libconfig_default, path, - value); - - /* Runtime option overrides default value */ - ret_rt = config_lookup_int(&odp_global_data.libconfig_runtime, path, - value); - - return (ret_def == CONFIG_TRUE || ret_rt == CONFIG_TRUE) ? 1 : 0; -} diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index e0d565dc2..2f3bea21b 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -1140,6 +1140,93 @@ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt) return dst_size < src_size; } +/* Simple implementation of ones complement sum. + * Based on RFC1071 and its errata. + */ +typedef union { + uint16_t w; + uint8_t b[2]; +} swap_buf_t; + +static uint32_t segment_sum16_32(const uint8_t *p, + uint32_t len, + uint32_t offset) + +{ + uint32_t sum = 0; + + /* Include second part of 16-bit short word split between segments */ + if (len > 0 && (offset % 2)) { + swap_buf_t sw; + + sw.b[0] = 0; + sw.b[1] = *p++; + sum = sw.w; + len--; + } + + /* + * If pointer is 16-bit aligned, we can do fast path calculation. + * If it is not, we sum hi and lo bytes separately and then sum them. + */ + if ((uintptr_t)p % 2) { + uint32_t sum1 = 0, sum2 = 0; + + while (len > 1) { + sum1 += *p++; + sum2 += *p++; + len -= 2; + } +#if (ODP_BYTE_ORDER == ODP_BIG_ENDIAN) + sum += sum2 + (sum1 << 8); +#else + sum += sum1 + (sum2 << 8); +#endif + } else { + while (len > 1) { + sum += *(const uint16_t *)(uintptr_t)p; + p += 2; + len -= 2; + } + } + + /* Add left-over byte, if any */ + if (len > 0) { + swap_buf_t sw; + + sw.b[0] = *p; + sw.b[1] = 0; + sum += sw.w; + } + + return sum; +} + +static uint32_t packet_sum16_32(odp_packet_hdr_t *pkt_hdr, + uint32_t offset, + uint32_t len) +{ + uint32_t sum = 0; + + if (offset + len > packet_len(pkt_hdr)) + return 0; + + while (len > 0) { + uint32_t seglen = 0; /* GCC */ + void *mapaddr = _odp_packet_offset(packet_handle(pkt_hdr), + offset, &seglen, NULL); + + if (seglen > len) + seglen = len; + + sum += segment_sum16_32(mapaddr, seglen, offset); + len -= seglen; + offset += seglen; + } + + return sum; +} + /** Parser helper function for Ethernet packets */ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len) @@ -1496,6 +1583,158 @@ int packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, seg_len, layer, ethtype); } +static inline int packet_ipv4_chksum(odp_packet_t pkt, + uint32_t offset, + _odp_ipv4hdr_t *ip, + odp_u16sum_t *chksum) +{ + unsigned int nleft = _ODP_IPV4HDR_IHL(ip->ver_ihl) * 4; + uint16_t buf[nleft / 2]; + int res; + + if (odp_unlikely(nleft < sizeof(*ip))) + return -1; + ip->chksum = 0; + memcpy(buf, ip, sizeof(*ip)); + res = odp_packet_copy_to_mem(pkt, offset + sizeof(*ip), + nleft - sizeof(*ip), + buf + sizeof(*ip) / 2); + if (odp_unlikely(res < 0)) + return res; + + *chksum = ~odp_chksum_ones_comp16(buf, nleft); + + return 0; +} + +#define _ODP_IPV4HDR_CSUM_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, chksum) +#define _ODP_IPV4ADDR_OFFSSET ODP_OFFSETOF(_odp_ipv4hdr_t, src_addr) +#define _ODP_IPV6ADDR_OFFSSET ODP_OFFSETOF(_odp_ipv6hdr_t, src_addr) +#define _ODP_IPV4HDR_CSUM_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, chksum) +#define _ODP_UDP_LEN_OFFSET ODP_OFFSETOF(_odp_udphdr_t, length) +#define _ODP_UDP_CSUM_OFFSET ODP_OFFSETOF(_odp_udphdr_t, chksum) + +/** + * Calculate and fill in IPv4 checksum + * + * @param pkt ODP packet + * + * @retval 0 on success + * @retval <0 on failure + */ +int _odp_packet_ipv4_chksum_insert(odp_packet_t pkt) +{ + uint32_t offset; + _odp_ipv4hdr_t ip; + odp_u16sum_t chksum; + int res; + + offset = odp_packet_l3_offset(pkt); + if (offset == ODP_PACKET_OFFSET_INVALID) + return -1; + + res = odp_packet_copy_to_mem(pkt, offset, sizeof(ip), &ip); + if (odp_unlikely(res < 0)) + return res; + + res = packet_ipv4_chksum(pkt, offset, &ip, &chksum); + if (odp_unlikely(res < 0)) + return res; + + return odp_packet_copy_from_mem(pkt, + offset + _ODP_IPV4HDR_CSUM_OFFSET, + 2, &chksum); +} + +static int _odp_packet_tcp_udp_chksum_insert(odp_packet_t pkt, uint16_t proto) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t zero = 0; + uint32_t sum; + uint16_t l3_ver; + uint16_t chksum; + uint32_t chksum_offset; + uint32_t frame_len = packet_len(pkt_hdr); + + if (pkt_hdr->p.l3_offset == ODP_PACKET_OFFSET_INVALID) + return -1; + if (pkt_hdr->p.l4_offset == ODP_PACKET_OFFSET_INVALID) + return -1; + + odp_packet_copy_to_mem(pkt, pkt_hdr->p.l3_offset, 2, &l3_ver); + + if (_ODP_IPV4HDR_VER(l3_ver) == _ODP_IPV4) + sum = packet_sum16_32(pkt_hdr, + pkt_hdr->p.l3_offset + + _ODP_IPV4ADDR_OFFSSET, + 2 * _ODP_IPV4ADDR_LEN); + else + sum = packet_sum16_32(pkt_hdr, + pkt_hdr->p.l3_offset + + _ODP_IPV6ADDR_OFFSSET, + 2 * _ODP_IPV6ADDR_LEN); +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN + sum += proto; +#else + sum += proto << 8; +#endif + + if (proto == _ODP_IPPROTO_TCP) { + sum += _odp_cpu_to_be_16(frame_len - pkt_hdr->p.l4_offset); + chksum_offset = pkt_hdr->p.l4_offset + _ODP_UDP_CSUM_OFFSET; + } else { + sum += packet_sum16_32(pkt_hdr, + pkt_hdr->p.l4_offset + + _ODP_UDP_LEN_OFFSET, + 2); + chksum_offset = pkt_hdr->p.l4_offset + _ODP_UDP_CSUM_OFFSET; + } + odp_packet_copy_from_mem(pkt, chksum_offset, 2, &zero); + + sum += packet_sum16_32(pkt_hdr, + pkt_hdr->p.l4_offset, + frame_len - pkt_hdr->p.l4_offset); + + /* Not more than two additions */ + sum = (sum & 0xffff) + (sum >> 16); + sum = (sum & 0xffff) + (sum >> 16); + + chksum = ~sum; + + if (proto == _ODP_IPPROTO_UDP && chksum == 0) + chksum = 0xffff; + + return odp_packet_copy_from_mem(pkt, + chksum_offset, + 2, &chksum); +} + +/** + * Calculate and fill in TCP checksum + * + * @param pkt ODP packet + * + * @retval 0 on success + * @retval <0 on failure + */ +int _odp_packet_tcp_chksum_insert(odp_packet_t pkt) +{ + return _odp_packet_tcp_udp_chksum_insert(pkt, _ODP_IPPROTO_TCP); +} + +/** + * Calculate and fill in UDP checksum + * + * @param pkt ODP packet + * + * @retval 0 on success + * @retval <0 on failure + */ +int _odp_packet_udp_chksum_insert(odp_packet_t pkt) +{ + return _odp_packet_tcp_udp_chksum_insert(pkt, _ODP_IPPROTO_UDP); +} + /** * Simple packet parser */ diff --git a/platform/linux-dpdk/odp_packet_dpdk.c b/platform/linux-dpdk/odp_packet_dpdk.c index 1c85577fa..eae2da1dc 100644 --- a/platform/linux-dpdk/odp_packet_dpdk.c +++ b/platform/linux-dpdk/odp_packet_dpdk.c @@ -32,9 +32,17 @@ #include <odp_classification_internal.h> #include <odp_packet_io_internal.h> #include <odp_libconfig_internal.h> -#include <odp_packet_dpdk.h> +#include <odp/api/plat/packet_inlines.h> #include <net/if.h> +#include <protocols/udp.h> + +#include <rte_config.h> +#include <rte_ethdev.h> +#include <rte_ip_frag.h> +#include <rte_udp.h> +#include <rte_tcp.h> + /* DPDK poll mode drivers requiring minimum RX burst size DPDK_MIN_RX_BURST */ #define IXGBE_DRV_NAME "net_ixgbe" #define I40E_DRV_NAME "net_i40e" @@ -57,22 +65,16 @@ extern void *pktio_entry_ptr[ODP_CONFIG_PKTIO_ENTRIES]; static uint32_t mtu_get_pkt_dpdk(pktio_entry_t *pktio_entry); -static int lookup_opt(const char *path, const char *drv_name, int *val) +static int lookup_opt(const char *opt_name, const char *drv_name, int *val) { const char *base = "pktio_dpdk"; - char opt_path[256]; - int ret = 0; - - /* Default option */ - snprintf(opt_path, sizeof(opt_path), "%s.%s", base, path); - ret += _odp_libconfig_lookup_int(opt_path, val); - - /* Driver specific option overrides default option */ - snprintf(opt_path, sizeof(opt_path), "%s.%s.%s", base, drv_name, path); - ret += _odp_libconfig_lookup_int(opt_path, val); + int ret; + ret = _odp_libconfig_lookup_ext_int(base, drv_name, opt_name, val); if (ret == 0) - ODP_ERR("Unable to find DPDK configuration option: %s\n", path); + ODP_ERR("Unable to find DPDK configuration option: %s\n", + opt_name); + return ret; } @@ -385,8 +387,9 @@ static int setup_pkt_dpdk(odp_pktio_t pktio ODP_UNUSED, return 0; } -static int close_pkt_dpdk(pktio_entry_t *pktio_entry ODP_UNUSED) +static int close_pkt_dpdk(pktio_entry_t *pktio_entry) { + rte_eth_dev_stop(pktio_entry->s.pkt_dpdk.port_id); return 0; } @@ -441,6 +444,10 @@ static int start_pkt_dpdk(pktio_entry_t *pktio_entry) struct rte_eth_txconf *txconf = NULL; uint32_t txq_flags = 0; + if (pktio_entry->s.state == PKTIO_STATE_STOPPED || + pktio_entry->s.state == PKTIO_STATE_STOP_PENDING) + rte_eth_dev_stop(pkt_dpdk->port_id); + /* DPDK doesn't support nb_rx_q/nb_tx_q being 0 */ if (!pktio_entry->s.num_in_queue) pktio_entry->s.num_in_queue = 1; @@ -548,7 +555,14 @@ static int start_pkt_dpdk(pktio_entry_t *pktio_entry) static int stop_pkt_dpdk(pktio_entry_t *pktio_entry) { - rte_eth_dev_stop(pktio_entry->s.pkt_dpdk.port_id); + unsigned int i; + uint16_t port_id = pktio_entry->s.pkt_dpdk.port_id; + + for (i = 0; i < pktio_entry->s.num_in_queue; i++) + rte_eth_dev_rx_queue_stop(port_id, i); + for (i = 0; i < pktio_entry->s.num_out_queue; i++) + rte_eth_dev_tx_queue_stop(port_id, i); + return 0; } @@ -590,6 +604,7 @@ static void _odp_pktio_send_completion(pktio_entry_t *pktio_entry) #define IP4_CSUM_RESULT(m) (m->ol_flags & PKT_RX_IP_CKSUM_MASK) #define L4_CSUM_RESULT(m) (m->ol_flags & PKT_RX_L4_CKSUM_MASK) #define HAS_L4_PROTO(m, proto) ((m->packet_type & RTE_PTYPE_L4_MASK) == proto) +#define UDP4_CSUM(_p) (((_odp_udphdr_t *)_odp_packet_l4_ptr(_p, NULL))->chksum) #define PKTIN_CSUM_BITS 0x1C @@ -622,6 +637,13 @@ static inline int pkt_set_ol_rx(odp_pktin_config_opt_t *pktin_cfg, if (packet_csum_result == PKT_RX_L4_CKSUM_GOOD) { pkt_hdr->p.input_flags.l4_chksum_done = 1; } else if (packet_csum_result != PKT_RX_L4_CKSUM_UNKNOWN) { + if (pkt_hdr->p.input_flags.ipv4 && + pkt_hdr->p.input_flags.udp && + !UDP4_CSUM(packet_handle(pkt_hdr))) { + pkt_hdr->p.input_flags.l4_chksum_done = 1; + return 0; + } + if (pktin_cfg->bit.drop_udp_err) return -1; @@ -860,11 +882,7 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg, if (!ipv4_chksum_pkt && !udp_chksum_pkt && !tcp_chksum_pkt) return; - if (pkt_p->l4_offset == ODP_PACKET_OFFSET_INVALID) - return; - mbuf->l2_len = pkt_p->l3_offset - pkt_p->l2_offset; - mbuf->l3_len = pkt_p->l4_offset - pkt_p->l3_offset; if (l3_proto_v4) mbuf->ol_flags = PKT_TX_IPV4; @@ -875,8 +893,14 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg, mbuf->ol_flags |= PKT_TX_IP_CKSUM; ((struct ipv4_hdr *)l3_hdr)->hdr_checksum = 0; + mbuf->l3_len = _ODP_IPV4HDR_IHL(*(uint8_t *)l3_hdr) * 4; } + if (pkt_p->l4_offset == ODP_PACKET_OFFSET_INVALID) + return; + + mbuf->l3_len = pkt_p->l4_offset - pkt_p->l3_offset; + l4_hdr = (void *)(mbuf_data + pkt_p->l4_offset); if (udp_chksum_pkt) { diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index 463d7aedb..d5b3195a9 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -29,8 +29,7 @@ #include <odp/api/plat/pool_inline_types.h> -/* for DPDK */ -#include <odp_packet_dpdk.h> +#include <rte_config.h> #include <rte_version.h> #ifdef POOL_USE_TICKETLOCK diff --git a/platform/linux-dpdk/odp_queue_basic.c b/platform/linux-dpdk/odp_queue_basic.c index 5df48ecc2..67303c185 100644 --- a/platform/linux-dpdk/odp_queue_basic.c +++ b/platform/linux-dpdk/odp_queue_basic.c @@ -25,6 +25,7 @@ #include <odp/api/hints.h> #include <odp/api/sync.h> #include <odp/api/traffic_mngr.h> +#include <odp_libconfig_internal.h> #define NUM_INTERNAL_QUEUES 64 @@ -36,6 +37,9 @@ #include <string.h> #include <inttypes.h> +#define MIN_QUEUE_SIZE 8 +#define MAX_QUEUE_SIZE (1 * 1024 * 1024) + static int queue_init(queue_entry_t *queue, const char *name, const odp_queue_param_t *param); @@ -60,11 +64,11 @@ static int queue_capa(odp_queue_capability_t *capa, int sched) /* Reserve some queues for internal use */ capa->max_queues = ODP_CONFIG_QUEUES - NUM_INTERNAL_QUEUES; capa->plain.max_num = capa->max_queues; - capa->plain.max_size = CONFIG_QUEUE_SIZE - 1; + capa->plain.max_size = queue_glb->config.max_queue_size - 1; capa->plain.lockfree.max_num = queue_glb->queue_lf_num; capa->plain.lockfree.max_size = queue_glb->queue_lf_size; capa->sched.max_num = capa->max_queues; - capa->sched.max_size = CONFIG_QUEUE_SIZE - 1; + capa->sched.max_size = queue_glb->config.max_queue_size - 1; if (sched) { capa->max_ordered_locks = sched_fn->max_ordered_locks(); @@ -75,6 +79,52 @@ static int queue_capa(odp_queue_capability_t *capa, int sched) return 0; } +static int read_config_file(queue_global_t *queue_glb) +{ + const char *str; + uint32_t val_u32; + int val = 0; + + ODP_PRINT("Queue config:\n"); + + str = "queue_basic.max_queue_size"; + if (!_odp_libconfig_lookup_int(str, &val)) { + ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + + val_u32 = val; + + if (val_u32 > MAX_QUEUE_SIZE || val_u32 < MIN_QUEUE_SIZE || + !CHECK_IS_POWER2(val_u32)) { + ODP_ERR("Bad value %s = %u\n", str, val_u32); + return -1; + } + + queue_glb->config.max_queue_size = val_u32; + ODP_PRINT(" %s: %u\n", str, val_u32); + + str = "queue_basic.default_queue_size"; + if (!_odp_libconfig_lookup_int(str, &val)) { + ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + + val_u32 = val; + + if (val_u32 > queue_glb->config.max_queue_size || + val_u32 < MIN_QUEUE_SIZE || + !CHECK_IS_POWER2(val_u32)) { + ODP_ERR("Bad value %s = %u\n", str, val_u32); + return -1; + } + + queue_glb->config.default_queue_size = val_u32; + ODP_PRINT(" %s: %u\n\n", str, val_u32); + + return 0; +} + static int queue_init_global(void) { uint32_t i; @@ -85,7 +135,7 @@ static int queue_init_global(void) ODP_DBG("Starts...\n"); - shm = odp_shm_reserve("odp_queues", + shm = odp_shm_reserve("_odp_queue_gbl", sizeof(queue_global_t), sizeof(queue_entry_t), 0); @@ -105,6 +155,15 @@ static int queue_init_global(void) queue->s.handle = queue_from_index(i); } + if (read_config_file(queue_glb)) { + odp_shm_free(shm); + return -1; + } + + queue_glb->queue_gbl_shm = shm; + queue_glb->queue_ring_shm = ODP_SHM_INVALID; + queue_glb->ring_data = NULL; + lf_func = &queue_glb->queue_lf_func; queue_glb->queue_lf_num = queue_lf_init_global(&lf_size, lf_func); queue_glb->queue_lf_size = lf_size; @@ -134,7 +193,6 @@ static int queue_term_local(void) static int queue_term_global(void) { int ret = 0; - int rc = 0; queue_entry_t *queue; int i; @@ -143,20 +201,19 @@ static int queue_term_global(void) LOCK(queue); if (queue->s.status != QUEUE_STATUS_FREE) { ODP_ERR("Not destroyed queue: %s\n", queue->s.name); - rc = -1; + ret = -1; } UNLOCK(queue); } queue_lf_term_global(); - ret = odp_shm_free(odp_shm_lookup("odp_queues")); - if (ret < 0) { - ODP_ERR("shm free failed for odp_queues"); - rc = -1; + if (odp_shm_free(queue_glb->queue_gbl_shm)) { + ODP_ERR("shm free failed"); + ret = -1; } - return rc; + return ret; } static int queue_capability(odp_queue_capability_t *capa) @@ -208,7 +265,7 @@ static odp_queue_t queue_create(const char *name, } if (param->nonblocking == ODP_BLOCKING) { - if (param->size > CONFIG_QUEUE_SIZE) + if (param->size > queue_glb->config.max_queue_size) return ODP_QUEUE_INVALID; } else if (param->nonblocking == ODP_NONBLOCKING_LF) { /* Only plain type lock-free queues supported */ @@ -558,6 +615,8 @@ static odp_event_t queue_deq(odp_queue_t handle) static int queue_init(queue_entry_t *queue, const char *name, const odp_queue_param_t *param) { + uint32_t queue_size; + if (name == NULL) { queue->s.name[0] = 0; } else { @@ -581,7 +640,21 @@ static int queue_init(queue_entry_t *queue, const char *name, queue->s.pktin = PKTIN_INVALID; queue->s.pktout = PKTOUT_INVALID; - queue->s.ring_st = ring_st_create(queue->s.name, CONFIG_QUEUE_SIZE); + /* Use default size for all small queues to quarantee performance + * level. */ + queue_size = queue_glb->config.default_queue_size; + if (param->size > queue_glb->config.default_queue_size) + queue_size = param->size; + + /* Round up if not already a power of two */ + queue_size = ROUNDUP_POWER2_U32(queue_size); + + if (queue_size > queue_glb->config.max_queue_size) { + ODP_ERR("Too large queue size %u\n", queue_size); + return -1; + } + + queue->s.ring_st = ring_st_create(queue->s.name, queue_size); if (queue->s.ring_st == NULL) return -1; diff --git a/platform/linux-dpdk/odp_time.c b/platform/linux-dpdk/odp_time.c index 3cbb771d8..e4c82b756 100644 --- a/platform/linux-dpdk/odp_time.c +++ b/platform/linux-dpdk/odp_time.c @@ -11,10 +11,12 @@ #include <odp/api/hints.h> #include <odp_debug_internal.h> #include <odp_arch_time_internal.h> -#include <rte_cycles.h> #include <string.h> #include <inttypes.h> +#include <rte_config.h> +#include <rte_cycles.h> + typedef uint64_t (*time_to_ns_fn) (odp_time_t time); typedef odp_time_t (*time_cur_fn)(void); typedef odp_time_t (*time_from_ns_fn) (uint64_t ns); diff --git a/platform/linux-dpdk/test/validation/api/pktio/pktio_run.sh b/platform/linux-dpdk/test/validation/api/pktio/pktio_run.sh index 6007195e1..d051fa801 100755 --- a/platform/linux-dpdk/test/validation/api/pktio/pktio_run.sh +++ b/platform/linux-dpdk/test/validation/api/pktio/pktio_run.sh @@ -85,7 +85,7 @@ run() echo "Failed to setup test environment, skipping test." exit $TEST_SKIPPED fi - export ODP_PLATFORM_PARAMS="-n 4 --no-pci --vdev net_pcap0,iface=$IF0 --vdev net_pcap1,iface=$IF1" + export ODP_PLATFORM_PARAMS="--no-pci --vdev net_pcap0,iface=$IF0 --vdev net_pcap1,iface=$IF1" export ODP_PKTIO_IF0=0 export ODP_PKTIO_IF1=1 fi diff --git a/platform/linux-dpdk/test/wrapper-script.sh b/platform/linux-dpdk/test/wrapper-script.sh index 23ebec202..b092e1064 100755 --- a/platform/linux-dpdk/test/wrapper-script.sh +++ b/platform/linux-dpdk/test/wrapper-script.sh @@ -1,6 +1,6 @@ #!/bin/bash -export ODP_PLATFORM_PARAMS=${ODP_PLATFORM_PARAMS:--n 4 --vdev "crypto_openssl" --vdev crypto_null} +export ODP_PLATFORM_PARAMS=${ODP_PLATFORM_PARAMS:---vdev="crypto_openssl" --vdev="crypto_null"} # where to mount huge pages export HUGEPAGEDIR=${HUGEPAGEDIR:-/mnt/huge} # exit codes expected by automake for skipped tests @@ -52,21 +52,14 @@ if [ "$(id -u)" != "0" ]; then fi echo "Mounting hugetlbfs" -export SIZE=1G -export SIZE_KB=1048576 -export RESERVE=1 +export SIZE=2MB +export SIZE_KB=2048 +export RESERVE=512 mount_and_reserve res=$? if [ $res -ne 0 ]; then - export SIZE=2MB - export SIZE_KB=2048 - export RESERVE=1024 - mount_and_reserve - res=$? - if [ $res -ne 0 ]; then - echo "ERROR: can't mount hugepages with any size" - exit $res - fi + echo "ERROR: can't mount hugepages" + exit $res fi echo "running $1!" $1 diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 7e57994d1..4f03eab15 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -82,15 +82,11 @@ endif noinst_HEADERS = \ arch/odp_arch_time_internal.h \ - include/_fdserver_internal.h \ - include/_ishm_internal.h \ - include/_ishmphy_internal.h \ - include/_ishmpool_internal.h \ include/odp_align_internal.h \ include/odp_atomic_internal.h \ - include/odp_buffer_inlines.h \ include/odp_bitmap_internal.h \ include/odp_bitset.h \ + include/odp_buffer_inlines.h \ include/odp_buffer_internal.h \ include/odp_classification_datamodel.h \ include/odp_classification_inlines.h \ @@ -98,37 +94,41 @@ noinst_HEADERS = \ include/odp_config_internal.h \ include/odp_debug_internal.h \ include/odp_errno_define.h \ + include/odp_fdserver_internal.h \ include/odp_forward_typedefs_internal.h \ include/odp_internal.h \ include/odp_ipsec_internal.h \ + include/odp_ishm_internal.h \ + include/odp_ishmphy_internal.h \ + include/odp_ishmpool_internal.h \ include/odp_libconfig_internal.h \ include/odp_llqueue.h \ include/odp_macros_internal.h \ include/odp_name_table_internal.h \ + include/odp_packet_dpdk.h \ include/odp_packet_internal.h \ include/odp_packet_io_internal.h \ include/odp_packet_io_ipc_internal.h \ include/odp_packet_io_ring_internal.h \ include/odp_packet_netmap.h \ - include/odp_packet_dpdk.h \ + include/odp_packet_null.h \ include/odp_packet_socket.h \ include/odp_packet_tap.h \ - include/odp_packet_null.h \ include/odp_pkt_queue_internal.h \ include/odp_pool_internal.h \ include/odp_posix_extensions.h \ - include/odp_queue_internal.h \ - include/odp_queue_scalable_internal.h \ include/odp_queue_if.h \ + include/odp_queue_internal.h \ include/odp_queue_lf.h \ + include/odp_queue_scalable_internal.h \ include/odp_ring_internal.h \ include/odp_ring_st_internal.h \ include/odp_schedule_if.h \ - include/odp_schedule_scalable.h \ include/odp_schedule_scalable_config.h \ + include/odp_schedule_scalable.h \ include/odp_schedule_scalable_ordered.h \ - include/odp_sorted_list_internal.h \ include/odp_shm_internal.h \ + include/odp_sorted_list_internal.h \ include/odp_timer_internal.h \ include/odp_timer_wheel_internal.h \ include/odp_traffic_mngr_internal.h \ @@ -138,15 +138,10 @@ noinst_HEADERS = \ include/protocols/tcp.h \ include/protocols/thash.h \ include/protocols/udp.h - nodist_noinst_HEADERS = \ include/odp_libconfig_config.h __LIB__libodp_linux_la_SOURCES = \ - _fdserver.c \ - _ishm.c \ - _ishmphy.c \ - _ishmpool.c \ odp_atomic.c \ odp_barrier.c \ odp_bitmap.c \ @@ -159,30 +154,21 @@ __LIB__libodp_linux_la_SOURCES = \ odp_crypto.c \ odp_errno.c \ odp_event.c \ + odp_fdserver.c \ odp_hash.c \ - odp_init.c \ odp_impl.c \ + odp_init.c \ odp_ipsec.c \ odp_ipsec_events.c \ odp_ipsec_sad.c \ + odp_ishm.c \ + odp_ishmphy.c \ + odp_ishmpool.c \ odp_libconfig.c \ odp_name_table.c \ odp_packet.c \ odp_packet_flags.c \ odp_packet_io.c \ - pktio/ethtool.c \ - pktio/io_ops.c \ - pktio/ipc.c \ - pktio/pktio_common.c \ - pktio/loop.c \ - pktio/netmap.c \ - pktio/null.c \ - pktio/dpdk.c \ - pktio/socket.c \ - pktio/socket_mmap.c \ - pktio/sysfs.c \ - pktio/tap.c \ - pktio/ring.c \ odp_pkt_queue.c \ odp_pool.c \ odp_queue_basic.c \ @@ -193,10 +179,10 @@ __LIB__libodp_linux_la_SOURCES = \ odp_rwlock_recursive.c \ odp_schedule_basic.c \ odp_schedule_if.c \ - odp_schedule_sp.c \ odp_schedule_iquery.c \ odp_schedule_scalable.c \ odp_schedule_scalable_ordered.c \ + odp_schedule_sp.c \ odp_shared_memory.c \ odp_sorted_list.c \ odp_spinlock.c \ @@ -209,8 +195,20 @@ __LIB__libodp_linux_la_SOURCES = \ odp_timer_wheel.c \ odp_traffic_mngr.c \ odp_version.c \ - odp_weak.c - + odp_weak.c \ + pktio/dpdk.c \ + pktio/ethtool.c \ + pktio/io_ops.c \ + pktio/ipc.c \ + pktio/loop.c \ + pktio/netmap.c \ + pktio/null.c \ + pktio/pktio_common.c \ + pktio/ring.c \ + pktio/socket.c \ + pktio/socket_mmap.c \ + pktio/sysfs.c \ + pktio/tap.c if ODP_ABI_COMPAT __LIB__libodp_linux_la_SOURCES += \ odp_atomic_api.c \ diff --git a/platform/linux-generic/include/_fdserver_internal.h b/platform/linux-generic/include/odp_fdserver_internal.h index 8518a5b7e..8518a5b7e 100644 --- a/platform/linux-generic/include/_fdserver_internal.h +++ b/platform/linux-generic/include/odp_fdserver_internal.h diff --git a/platform/linux-generic/include/_ishm_internal.h b/platform/linux-generic/include/odp_ishm_internal.h index 56c7f5a93..56c7f5a93 100644 --- a/platform/linux-generic/include/_ishm_internal.h +++ b/platform/linux-generic/include/odp_ishm_internal.h diff --git a/platform/linux-generic/include/_ishmphy_internal.h b/platform/linux-generic/include/odp_ishmphy_internal.h index 05e3fcec7..05e3fcec7 100644 --- a/platform/linux-generic/include/_ishmphy_internal.h +++ b/platform/linux-generic/include/odp_ishmphy_internal.h diff --git a/platform/linux-generic/include/_ishmpool_internal.h b/platform/linux-generic/include/odp_ishmpool_internal.h index 94bcddaeb..94bcddaeb 100644 --- a/platform/linux-generic/include/_ishmpool_internal.h +++ b/platform/linux-generic/include/odp_ishmpool_internal.h diff --git a/platform/linux-generic/include/odp_libconfig_internal.h b/platform/linux-generic/include/odp_libconfig_internal.h index 042917755..727f68863 100644 --- a/platform/linux-generic/include/odp_libconfig_internal.h +++ b/platform/linux-generic/include/odp_libconfig_internal.h @@ -22,6 +22,11 @@ int _odp_libconfig_term_global(void); int _odp_libconfig_lookup_int(const char *path, int *value); +int _odp_libconfig_lookup_ext_int(const char *base_path, + const char *local_path, + const char *name, + int *value); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 75b4ce9e5..be17cac86 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -303,6 +303,10 @@ int _odp_packet_set_data(odp_packet_t pkt, uint32_t offset, int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset, const void *s, uint32_t len); +int _odp_packet_ipv4_chksum_insert(odp_packet_t pkt); +int _odp_packet_tcp_chksum_insert(odp_packet_t pkt); +int _odp_packet_udp_chksum_insert(odp_packet_t pkt); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_packet_netmap.h b/platform/linux-generic/include/odp_packet_netmap.h index bb81f5f85..bd3efdcf5 100644 --- a/platform/linux-generic/include/odp_packet_netmap.h +++ b/platform/linux-generic/include/odp_packet_netmap.h @@ -19,6 +19,12 @@ #define NM_MAX_DESC 64 +/** netmap runtime configuration options */ +typedef struct { + int nr_rx_slots; + int nr_tx_slots; +} netmap_opt_t; + /** Ring for mapping pktin/pktout queues to netmap descriptors */ struct netmap_ring_t { unsigned first; /**< Index of first netmap descriptor */ @@ -61,6 +67,7 @@ typedef struct { netmap_ring_t rx_desc_ring[PKTIO_MAX_QUEUES]; /** mapping of pktout queues to netmap tx descriptors */ netmap_ring_t tx_desc_ring[PKTIO_MAX_QUEUES]; + netmap_opt_t opt; /**< options */ } pkt_netmap_t; #endif diff --git a/platform/linux-generic/include/odp_queue_internal.h b/platform/linux-generic/include/odp_queue_internal.h index 3aec3fe9d..98e86fa0e 100644 --- a/platform/linux-generic/include/odp_queue_internal.h +++ b/platform/linux-generic/include/odp_queue_internal.h @@ -63,17 +63,19 @@ union queue_entry_u { uint8_t pad[ROUNDUP_CACHE_LINE(sizeof(struct queue_entry_s))]; }; -typedef struct ODP_ALIGNED_CACHE { - /* Storage space for ring data */ - uint32_t data[CONFIG_QUEUE_SIZE]; -} queue_ring_data_t; - typedef struct queue_global_t { - queue_entry_t queue[ODP_CONFIG_QUEUES]; - queue_ring_data_t ring_data[ODP_CONFIG_QUEUES]; - uint32_t queue_lf_num; - uint32_t queue_lf_size; - queue_lf_func_t queue_lf_func; + queue_entry_t queue[ODP_CONFIG_QUEUES]; + uint32_t *ring_data; + uint32_t queue_lf_num; + uint32_t queue_lf_size; + queue_lf_func_t queue_lf_func; + odp_shm_t queue_gbl_shm; + odp_shm_t queue_ring_shm; + + struct { + uint32_t max_queue_size; + uint32_t default_queue_size; + } config; } queue_global_t; diff --git a/platform/linux-generic/include/odp_schedule_scalable_ordered.h b/platform/linux-generic/include/odp_schedule_scalable_ordered.h index fb4720a51..17d4f7eab 100644 --- a/platform/linux-generic/include/odp_schedule_scalable_ordered.h +++ b/platform/linux-generic/include/odp_schedule_scalable_ordered.h @@ -14,7 +14,7 @@ #include <odp_internal.h> #include <odp_align_internal.h> #include <odp_bitset.h> -#include <_ishmpool_internal.h> +#include <odp_ishmpool_internal.h> /* High level functioning of reordering * Datastructures - diff --git a/platform/linux-generic/odp_crypto.c b/platform/linux-generic/odp_crypto.c index a7fddb5b4..852f02125 100644 --- a/platform/linux-generic/odp_crypto.c +++ b/platform/linux-generic/odp_crypto.c @@ -35,6 +35,8 @@ #endif #define MAX_SESSIONS 32 +#define AES_BLOCK_SIZE 16 +#define AES_KEY_LENGTH 16 /* * Cipher algorithm capabilities @@ -95,10 +97,18 @@ static const odp_crypto_auth_capability_t auth_capa_sha256_hmac[] = { {.digest_len = 16, .key_len = 32, .aad_len = {.min = 0, .max = 0, .inc = 0} }, {.digest_len = 32, .key_len = 32, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; +static const odp_crypto_auth_capability_t auth_capa_sha384_hmac[] = { +{.digest_len = 24, .key_len = 48, .aad_len = {.min = 0, .max = 0, .inc = 0} }, +{.digest_len = 48, .key_len = 48, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; + static const odp_crypto_auth_capability_t auth_capa_sha512_hmac[] = { {.digest_len = 32, .key_len = 64, .aad_len = {.min = 0, .max = 0, .inc = 0} }, {.digest_len = 64, .key_len = 64, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; +static const odp_crypto_auth_capability_t auth_capa_aes_xcbc[] = { +{.digest_len = 12, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0} }, +{.digest_len = 16, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; + static const odp_crypto_auth_capability_t auth_capa_aes_gcm[] = { {.digest_len = 16, .key_len = 0, .aad_len = {.min = 8, .max = 12, .inc = 4} } }; @@ -308,6 +318,169 @@ void packet_hmac(odp_packet_t pkt, HMAC_Final(ctx, hash, NULL); } +static void xor_block(uint8_t *res, const uint8_t *op) +{ + int i; + + for (i = 0; i < AES_BLOCK_SIZE; i++) + res[i] ^= op[i]; +} + +static void memxor(uint8_t *res, const uint8_t *op, size_t len) +{ + for (size_t i = 0; i < len; i++) + res[i] ^= op[i]; +} + +static +void packet_aes_xcbc_mac(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session, + uint8_t *hash) +{ + uint8_t e[AES_BLOCK_SIZE] = {0}; + size_t eoff = 0; + uint32_t offset = param->auth_range.offset; + uint32_t len = param->auth_range.length; + uint32_t seglen = 0; + uint32_t datalen = 0; + int dummy_len = 0; + EVP_CIPHER_CTX *ctx; + void *mapaddr; + uint8_t *data = NULL; + + ODP_ASSERT(offset + len <= odp_packet_len(pkt)); + ODP_ASSERT(session != NULL); + ODP_ASSERT(sizeof(session->auth.key) >= 3 * AES_KEY_LENGTH); + + ctx = EVP_CIPHER_CTX_new(); + EVP_EncryptInit_ex(ctx, session->auth.evp_cipher, + NULL, session->auth.key, NULL); + + while (len > 0) { + mapaddr = odp_packet_offset(pkt, offset, &seglen, NULL); + datalen = seglen >= len ? len : seglen; + data = (uint8_t *)mapaddr; + offset += datalen; + len -= datalen; + if (eoff != 0) { + if (eoff + datalen > AES_BLOCK_SIZE) { + memxor(e + eoff, data, AES_BLOCK_SIZE - eoff); + datalen -= (AES_BLOCK_SIZE - eoff); + eoff = 0; + EVP_EncryptUpdate(ctx, + e, &dummy_len, e, sizeof(e)); + } else { + memxor(e + eoff, data, datalen); + eoff += datalen; + continue; + } + } + while (datalen > AES_BLOCK_SIZE) { + xor_block(e, data); + EVP_EncryptUpdate(ctx, e, &dummy_len, e, sizeof(e)); + data += AES_BLOCK_SIZE; + datalen -= AES_BLOCK_SIZE; + } + /* Segmentation handle */ + if (datalen > 0) { + memxor(e, data, datalen); + eoff = datalen; + } + } + + if (eoff == AES_BLOCK_SIZE) { + xor_block(e, session->auth.key + AES_KEY_LENGTH); + } else { + e[eoff] ^= 0x80; + xor_block(e, session->auth.key + AES_KEY_LENGTH * 2); + } + EVP_EncryptUpdate(ctx, hash, &dummy_len, e, sizeof(e)); + EVP_CIPHER_CTX_free(ctx); +} + +static +odp_crypto_alg_err_t auth_xcbcmac_gen(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + uint8_t hash[EVP_MAX_MD_SIZE]; + + /* Hash it */ + packet_aes_xcbc_mac(pkt, param, session, hash); + + /* Copy to the output location */ + odp_packet_copy_from_mem(pkt, + param->hash_result_offset, + session->p.auth_digest_len, + hash); + + return ODP_CRYPTO_ALG_ERR_NONE; +} + +static odp_crypto_alg_err_t +auth_xcbcmac_check(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + uint32_t bytes = session->p.auth_digest_len; + uint8_t hash_in[EVP_MAX_MD_SIZE]; + uint8_t hash_out[EVP_MAX_MD_SIZE]; + + /* Copy current value out and clear it before authentication */ + odp_packet_copy_to_mem(pkt, param->hash_result_offset, + bytes, hash_in); + + _odp_packet_set_data(pkt, param->hash_result_offset, + 0, bytes); + + /* Hash it */ + packet_aes_xcbc_mac(pkt, param, session, hash_out); + + /* Verify match */ + if (0 != memcmp(hash_in, hash_out, bytes)) + return ODP_CRYPTO_ALG_ERR_ICV_CHECK; + + /* Matched */ + return ODP_CRYPTO_ALG_ERR_NONE; +} + +static int process_aesxcbc_param(odp_crypto_generic_session_t *session, + const EVP_CIPHER *cipher) +{ + uint32_t k1[4] = { 0x01010101, 0x01010101, 0x01010101, 0x01010101 }; + uint32_t k2[4] = { 0x02020202, 0x02020202, 0x02020202, 0x02020202 }; + uint32_t k3[4] = { 0x03030303, 0x03030303, 0x03030303, 0x03030303 }; + EVP_CIPHER_CTX *ctx; + int dummy_len = 0; + + /* Set function */ + if (ODP_CRYPTO_OP_ENCODE == session->p.op) + session->auth.func = auth_xcbcmac_gen; + else + session->auth.func = auth_xcbcmac_check; + session->auth.init = null_crypto_init_routine; + + session->auth.evp_cipher = cipher; + ctx = EVP_CIPHER_CTX_new(); + EVP_EncryptInit_ex(ctx, session->auth.evp_cipher, NULL, + session->p.auth_key.data, NULL); + /* K1 = 0x01010101010101010101010101010101 encrypted with Key K */ + EVP_EncryptUpdate(ctx, session->auth.key, + &dummy_len, (uint8_t *)k1, AES_BLOCK_SIZE); + + /* K2 = 0x02020202020202020202020202020202 encrypted with Key K */ + EVP_EncryptUpdate(ctx, session->auth.key + AES_KEY_LENGTH, + &dummy_len, (uint8_t *)k2, AES_BLOCK_SIZE); + + /* K3 = 0x03030303030303030303030303030303 encrypted with Key K */ + EVP_EncryptUpdate(ctx, session->auth.key + AES_KEY_LENGTH * 2, + &dummy_len, (uint8_t *)k3, AES_BLOCK_SIZE); + + EVP_CIPHER_CTX_free(ctx); + return 0; +} + static odp_crypto_alg_err_t auth_hmac_gen(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, @@ -661,9 +834,8 @@ static int process_cipher_param(odp_crypto_generic_session_t *session, return -1; /* Verify IV len is correct */ - if (!((0 == session->p.cipher_iv.length) || - ((uint32_t)EVP_CIPHER_iv_length(cipher) == - session->p.cipher_iv.length))) + if ((uint32_t)EVP_CIPHER_iv_length(cipher) != + session->p.cipher_iv.length) return -1; session->cipher.evp_cipher = cipher; @@ -791,6 +963,10 @@ static int process_aes_gcm_param(odp_crypto_generic_session_t *session, session->p.cipher_key.length) return -1; + /* Verify IV len is correct */ + if (12 != session->p.cipher_iv.length) + return -1; + memcpy(session->cipher.key_data, session->p.cipher_key.data, session->p.cipher_key.length); @@ -902,6 +1078,10 @@ static int process_aes_gmac_param(odp_crypto_generic_session_t *session, session->p.auth_key.length) return -1; + /* Verify IV len is correct */ + if (12 != session->p.auth_iv.length) + return -1; + memcpy(session->auth.key, session->p.auth_key.data, session->p.auth_key.length); @@ -1058,6 +1238,11 @@ static int process_aes_ccm_param(odp_crypto_generic_session_t *session, session->p.cipher_key.length) return -1; + /* Verify IV len is correct */ + if (11 != session->p.cipher_iv.length && + 13 != session->p.cipher_iv.length) + return -1; + memcpy(session->cipher.key_data, session->p.cipher_key.data, session->p.cipher_key.length); @@ -1078,6 +1263,10 @@ static int process_aes_ccm_param(odp_crypto_generic_session_t *session, static int process_auth_hmac_param(odp_crypto_generic_session_t *session, const EVP_MD *evp_md) { + /* Verify IV len is correct */ + if (0 != session->p.auth_iv.length) + return -1; + /* Set function */ if (ODP_CRYPTO_OP_ENCODE == session->p.op) session->auth.func = auth_hmac_gen; @@ -1106,6 +1295,9 @@ static int process_auth_cmac_param(odp_crypto_generic_session_t *session, session->p.auth_key.length) return -1; + if (0 != session->p.auth_iv.length) + return -1; + /* Set function */ if (ODP_CRYPTO_OP_ENCODE == session->p.op) session->auth.func = auth_cmac_gen; @@ -1152,7 +1344,9 @@ int odp_crypto_capability(odp_crypto_capability_t *capa) capa->auths.bit.md5_hmac = 1; capa->auths.bit.sha1_hmac = 1; capa->auths.bit.sha256_hmac = 1; + capa->auths.bit.sha384_hmac = 1; capa->auths.bit.sha512_hmac = 1; + capa->auths.bit.aes_xcbc_mac = 1; capa->auths.bit.aes_gcm = 1; capa->auths.bit.aes_ccm = 1; capa->auths.bit.aes_gmac = 1; @@ -1249,10 +1443,18 @@ int odp_crypto_auth_capability(odp_auth_alg_t auth, src = auth_capa_sha256_hmac; num = sizeof(auth_capa_sha256_hmac) / size; break; + case ODP_AUTH_ALG_SHA384_HMAC: + src = auth_capa_sha384_hmac; + num = sizeof(auth_capa_sha384_hmac) / size; + break; case ODP_AUTH_ALG_SHA512_HMAC: src = auth_capa_sha512_hmac; num = sizeof(auth_capa_sha512_hmac) / size; break; + case ODP_AUTH_ALG_AES_XCBC_MAC: + src = auth_capa_aes_xcbc; + num = sizeof(auth_capa_aes_xcbc) / size; + break; case ODP_AUTH_ALG_AES_GCM: src = auth_capa_aes_gcm; num = sizeof(auth_capa_aes_gcm) / size; @@ -1462,9 +1664,15 @@ odp_crypto_session_create(odp_crypto_session_param_t *param, case ODP_AUTH_ALG_SHA256_HMAC: rc = process_auth_hmac_param(session, EVP_sha256()); break; + case ODP_AUTH_ALG_SHA384_HMAC: + rc = process_auth_hmac_param(session, EVP_sha384()); + break; case ODP_AUTH_ALG_SHA512_HMAC: rc = process_auth_hmac_param(session, EVP_sha512()); break; + case ODP_AUTH_ALG_AES_XCBC_MAC: + rc = process_aesxcbc_param(session, EVP_aes_128_ecb()); + break; #if ODP_DEPRECATED_API case ODP_AUTH_ALG_AES128_GCM: if (param->cipher_alg == ODP_CIPHER_ALG_AES128_GCM) diff --git a/platform/linux-generic/_fdserver.c b/platform/linux-generic/odp_fdserver.c index 065736f01..0e9fb0e4d 100644 --- a/platform/linux-generic/_fdserver.c +++ b/platform/linux-generic/odp_fdserver.c @@ -42,7 +42,7 @@ #include <odp/api/spinlock.h> #include <odp_internal.h> #include <odp_debug_internal.h> -#include <_fdserver_internal.h> +#include <odp_fdserver_internal.h> #include <sys/prctl.h> #include <signal.h> @@ -70,6 +70,14 @@ #define MAP_ANONYMOUS MAP_ANON #endif +#define FD_ODP_DEBUG_PRINT 0 + +#define FD_ODP_DBG(fmt, ...) \ + do { \ + if (FD_ODP_DEBUG_PRINT == 1) \ + ODP_DBG(fmt, ##__VA_ARGS__);\ + } while (0) + /* when accessing the client functions, clients should be mutexed: */ static odp_spinlock_t *client_lock; @@ -257,7 +265,9 @@ static int get_socket(void) remote.sun_family = AF_UNIX; strcpy(remote.sun_path, sockpath); len = strlen(remote.sun_path) + sizeof(remote.sun_family); - if (connect(s_sock, (struct sockaddr *)&remote, len) == -1) { + while (connect(s_sock, (struct sockaddr *)&remote, len) == -1) { + if (errno == EINTR) + continue; ODP_ERR("cannot connect to server: %s\n", strerror(errno)); close(s_sock); return -1; @@ -280,8 +290,8 @@ int _odp_fdserver_register_fd(fd_server_context_e context, uint64_t key, odp_spinlock_lock(client_lock); - ODP_DBG("FD client register: pid=%d key=%" PRIu64 ", fd=%d\n", - getpid(), key, fd_to_send); + FD_ODP_DBG("FD client register: pid=%d key=%" PRIu64 ", fd=%d\n", + getpid(), key, fd_to_send); s_sock = get_socket(); if (s_sock < 0) { @@ -326,8 +336,8 @@ int _odp_fdserver_deregister_fd(fd_server_context_e context, uint64_t key) odp_spinlock_lock(client_lock); - ODP_DBG("FD client deregister: pid=%d key=%" PRIu64 "\n", - getpid(), key); + FD_ODP_DBG("FD client deregister: pid=%d key=%" PRIu64 "\n", + getpid(), key); s_sock = get_socket(); if (s_sock < 0) { @@ -413,7 +423,7 @@ static int stop_server(void) odp_spinlock_lock(client_lock); - ODP_DBG("FD sending server stop request\n"); + FD_ODP_DBG("FD sending server stop request\n"); s_sock = get_socket(); if (s_sock < 0) { @@ -464,8 +474,8 @@ static int handle_request(int client_sock) fd_table[fd_table_nb_entries].context = context; fd_table[fd_table_nb_entries].key = key; fd_table[fd_table_nb_entries++].fd = fd; - ODP_DBG("storing {ctx=%d, key=%" PRIu64 "}->fd=%d\n", - context, key, fd); + FD_ODP_DBG("storing {ctx=%d, key=%" PRIu64 "}->fd=%d\n", + context, key, fd); } else { ODP_ERR("FD table full\n"); send_fdserver_msg(client_sock, FD_REGISTER_NACK, @@ -517,9 +527,9 @@ static int handle_request(int client_sock) for (i = 0; i < fd_table_nb_entries; i++) { if ((fd_table[i].context == context) && (fd_table[i].key == key)) { - ODP_DBG("drop {ctx=%d," - " key=%" PRIu64 "}->fd=%d\n", - context, key, fd_table[i].fd); + FD_ODP_DBG("drop {ctx=%d," + " key=%" PRIu64 "}->fd=%d\n", + context, key, fd_table[i].fd); close(fd_table[i].fd); fd_table[i] = fd_table[--fd_table_nb_entries]; send_fdserver_msg(client_sock, @@ -535,7 +545,7 @@ static int handle_request(int client_sock) break; case FD_SERVERSTOP_REQ: - ODP_DBG("Stoping FD server\n"); + FD_ODP_DBG("Stoping FD server\n"); return 1; default: @@ -559,8 +569,11 @@ static void wait_requests(int sock) addr_sz = sizeof(remote); c_socket = accept(sock, (struct sockaddr *)&remote, &addr_sz); if (c_socket == -1) { - ODP_ERR("wait_requests: %s\n", strerror(errno)); - return; + if (errno == EINTR) + continue; + + ODP_ERR("wait_requests: %s\n", strerror(errno)); + return; } if (handle_request(c_socket)) @@ -636,6 +649,34 @@ int _odp_fdserver_init_global(void) } if (server_pid == 0) { /*child */ + sigset_t sigset; + struct sigaction action; + + sigfillset(&sigset); + /* undefined if these are ignored, as per POSIX */ + sigdelset(&sigset, SIGFPE); + sigdelset(&sigset, SIGILL); + sigdelset(&sigset, SIGSEGV); + /* can not be masked */ + sigdelset(&sigset, SIGKILL); + sigdelset(&sigset, SIGSTOP); + /* these we want to handle */ + sigdelset(&sigset, SIGTERM); + if (sigprocmask(SIG_SETMASK, &sigset, NULL) == -1) { + ODP_ERR("Could not set signal mask"); + exit(1); + } + + /* set default handlers for those signals we can handle */ + memset(&action, 0, sizeof(action)); + action.sa_handler = SIG_DFL; + sigemptyset(&action.sa_mask); + action.sa_flags = 0; + sigaction(SIGFPE, &action, NULL); + sigaction(SIGILL, &action, NULL); + sigaction(SIGSEGV, &action, NULL); + sigaction(SIGTERM, &action, NULL); + /* TODO: pin the server on appropriate service cpu mask */ /* when (if) we can agree on the usage of service mask */ @@ -643,6 +684,12 @@ int _odp_fdserver_init_global(void) /* orphans being "adopted" by the init process... */ prctl(PR_SET_PDEATHSIG, SIGTERM); + res = setsid(); + if (res == -1) { + ODP_ERR("Could not setsid()"); + exit(1); + } + /* allocate the space for the file descriptor<->key table: */ fd_table = malloc(FDSERVER_MAX_ENTRIES * sizeof(fdentry_t)); if (!fd_table) { diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 1e90cea0f..782cbf45c 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -162,67 +162,10 @@ static odp_ipsec_packet_result_t *ipsec_pkt_result(odp_packet_t packet) return &packet_hdr(packet)->ipsec_ctx; } -static inline int _odp_ipv4_csum(odp_packet_t pkt, - uint32_t offset, - _odp_ipv4hdr_t *ip, - odp_u16sum_t *chksum) -{ - unsigned nleft = _ODP_IPV4HDR_IHL(ip->ver_ihl) * 4; - uint16_t buf[nleft / 2]; - int res; - - if (odp_unlikely(nleft < sizeof(*ip))) - return -1; - ip->chksum = 0; - memcpy(buf, ip, sizeof(*ip)); - res = odp_packet_copy_to_mem(pkt, offset + sizeof(*ip), - nleft - sizeof(*ip), - buf + sizeof(*ip) / 2); - if (odp_unlikely(res < 0)) - return res; - - *chksum = ~odp_chksum_ones_comp16(buf, nleft); - - return 0; -} - -#define _ODP_IPV4HDR_CSUM_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, chksum) #define _ODP_IPV4HDR_PROTO_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, proto) #define _ODP_IPV6HDR_NHDR_OFFSET ODP_OFFSETOF(_odp_ipv6hdr_t, next_hdr) #define _ODP_IPV6HDREXT_NHDR_OFFSET ODP_OFFSETOF(_odp_ipv6hdr_ext_t, next_hdr) -/** - * Calculate and fill in IPv4 checksum - * - * @param pkt ODP packet - * - * @retval 0 on success - * @retval <0 on failure - */ -static inline int _odp_ipv4_csum_update(odp_packet_t pkt) -{ - uint32_t offset; - _odp_ipv4hdr_t ip; - odp_u16sum_t chksum; - int res; - - offset = odp_packet_l3_offset(pkt); - if (offset == ODP_PACKET_OFFSET_INVALID) - return -1; - - res = odp_packet_copy_to_mem(pkt, offset, sizeof(ip), &ip); - if (odp_unlikely(res < 0)) - return res; - - res = _odp_ipv4_csum(pkt, offset, &ip, &chksum); - if (odp_unlikely(res < 0)) - return res; - - return odp_packet_copy_from_mem(pkt, - offset + _ODP_IPV4HDR_CSUM_OFFSET, - 2, &chksum); -} - #define ipv4_hdr_len(ip) (_ODP_IPV4HDR_IHL((ip)->ver_ihl) * 4) static const uint8_t ipsec_padding[255] = { @@ -815,7 +758,7 @@ static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, ipv4hdr->tot_len = _odp_cpu_to_be_16(state.ip_tot_len); else ipv4hdr->ttl -= ipsec_sa->dec_ttl; - _odp_ipv4_csum_update(pkt); + _odp_packet_ipv4_chksum_insert(pkt); } else if (state.is_ipv6 && odp_packet_len(pkt) > _ODP_IPV6HDR_LEN) { _odp_ipv6hdr_t *ipv6hdr = odp_packet_l3_ptr(pkt, NULL); @@ -883,6 +826,7 @@ static int ipsec_out_tunnel_parse_ipv4(ipsec_state_t *state, state->out_tunnel.ip_tos = ipv4hdr->tos; state->out_tunnel.ip_df = _ODP_IPV4HDR_FLAGS_DONT_FRAG(flags); state->out_tunnel.ip_flabel = 0; + state->ip_next_hdr = ipv4hdr->proto; return 0; } @@ -1238,7 +1182,7 @@ static int ipsec_out_esp(odp_packet_t *pkt, static void ipsec_out_esp_post(ipsec_state_t *state, odp_packet_t pkt) { if (state->is_ipv4) - _odp_ipv4_csum_update(pkt); + _odp_packet_ipv4_chksum_insert(pkt); } static int ipsec_out_ah(odp_packet_t *pkt, @@ -1343,7 +1287,7 @@ static void ipsec_out_ah_post(ipsec_state_t *state, odp_packet_t pkt) ipv4hdr->tos = state->ah_ipv4.tos; ipv4hdr->frag_offset = state->ah_ipv4.frag_offset; - _odp_ipv4_csum_update(pkt); + _odp_packet_ipv4_chksum_insert(pkt); } else { _odp_ipv6hdr_t *ipv6hdr = odp_packet_l3_ptr(pkt, NULL); @@ -1352,6 +1296,41 @@ static void ipsec_out_ah_post(ipsec_state_t *state, odp_packet_t pkt) } } +#define OL_TX_CHKSUM_PKT(_cfg, _proto, _ovr_set, _ovr) \ + (_proto && (_ovr_set ? _ovr : _cfg)) + +static void ipsec_out_checksums(odp_packet_t pkt, + ipsec_state_t *state) +{ + odp_bool_t ipv4_chksum_pkt, udp_chksum_pkt, tcp_chksum_pkt; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + odp_ipsec_outbound_config_t outbound = ipsec_config.outbound; + + ipv4_chksum_pkt = OL_TX_CHKSUM_PKT(outbound.chksum.inner_ipv4, + state->is_ipv4, + pkt_hdr->p.flags.l3_chksum_set, + pkt_hdr->p.flags.l3_chksum); + udp_chksum_pkt = OL_TX_CHKSUM_PKT(outbound.chksum.inner_udp, + state->ip_next_hdr == + _ODP_IPPROTO_UDP, + pkt_hdr->p.flags.l4_chksum_set, + pkt_hdr->p.flags.l4_chksum); + tcp_chksum_pkt = OL_TX_CHKSUM_PKT(outbound.chksum.inner_tcp, + state->ip_next_hdr == + _ODP_IPPROTO_TCP, + pkt_hdr->p.flags.l4_chksum_set, + pkt_hdr->p.flags.l4_chksum); + + if (ipv4_chksum_pkt) + _odp_packet_ipv4_chksum_insert(pkt); + + if (tcp_chksum_pkt) + _odp_packet_tcp_chksum_insert(pkt); + + if (udp_chksum_pkt) + _odp_packet_udp_chksum_insert(pkt); +} + static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, odp_ipsec_sa_t sa, odp_packet_t *pkt_out, @@ -1411,6 +1390,9 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, if (state.ip_tot_len + state.ip_offset != odp_packet_len(pkt)) rc = -1; + + if (rc == 0) + ipsec_out_checksums(pkt, &state); } else { if (state.is_ipv4) rc = ipsec_out_tunnel_parse_ipv4(&state, ipsec_sa); @@ -1428,6 +1410,8 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, goto err; } + ipsec_out_checksums(pkt, &state); + if (ipsec_sa->tun_ipv4) rc = ipsec_out_tunnel_ipv4(&pkt, &state, ipsec_sa, opt->flag.ip_param ? @@ -1500,7 +1484,7 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, else if (ODP_IPSEC_AH == ipsec_sa->proto) ipsec_out_ah_post(&state, pkt); - _odp_ipv4_csum_update(pkt); + _odp_packet_ipv4_chksum_insert(pkt); *pkt_out = pkt; return ipsec_sa; diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 05865eb3a..c21269694 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -220,6 +220,8 @@ uint32_t _odp_ipsec_cipher_iv_len(odp_cipher_alg_t cipher) #endif case ODP_CIPHER_ALG_AES_GCM: return 12; + case ODP_CIPHER_ALG_AES_CCM: + return 11; case ODP_CIPHER_ALG_CHACHA20_POLY1305: return 12; default: @@ -244,14 +246,20 @@ uint32_t _odp_ipsec_auth_digest_len(odp_auth_alg_t auth) #endif case ODP_AUTH_ALG_SHA256_HMAC: return 16; + case ODP_AUTH_ALG_SHA384_HMAC: + return 24; case ODP_AUTH_ALG_SHA512_HMAC: return 32; + case ODP_AUTH_ALG_AES_XCBC_MAC: + return 12; #if ODP_DEPRECATED_API case ODP_AUTH_ALG_AES128_GCM: #endif case ODP_AUTH_ALG_AES_GCM: case ODP_AUTH_ALG_AES_GMAC: return 16; + case ODP_AUTH_ALG_AES_CCM: + return 16; case ODP_AUTH_ALG_CHACHA20_POLY1305: return 16; default: @@ -411,6 +419,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) case ODP_CIPHER_ALG_AES128_GCM: #endif case ODP_CIPHER_ALG_AES_GCM: + case ODP_CIPHER_ALG_AES_CCM: ipsec_sa->use_counter_iv = 1; ipsec_sa->esp_iv_len = 8; ipsec_sa->esp_block_len = 16; diff --git a/platform/linux-generic/_ishm.c b/platform/linux-generic/odp_ishm.c index ab112acea..babf51b91 100644 --- a/platform/linux-generic/_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -57,10 +57,10 @@ #include <odp_shm_internal.h> #include <odp_debug_internal.h> #include <odp_align_internal.h> -#include <_fdserver_internal.h> -#include <_ishm_internal.h> -#include <_ishmphy_internal.h> -#include <_ishmpool_internal.h> +#include <odp_fdserver_internal.h> +#include <odp_ishm_internal.h> +#include <odp_ishmphy_internal.h> +#include <odp_ishmpool_internal.h> #include <stdlib.h> #include <stdio.h> #include <unistd.h> @@ -456,10 +456,7 @@ static int create_file(int block_index, huge_flag_t huge, uint64_t len, fd = open(filename, oflag, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if (fd < 0) { - if (huge == HUGE) - ODP_DBG("open failed for %s: %s.\n", - filename, strerror(errno)); - else + if (huge != HUGE) ODP_ERR("open failed for %s: %s.\n", filename, strerror(errno)); return -1; @@ -569,7 +566,7 @@ static void *do_map(int block_index, uint64_t len, uint32_t align, } return NULL; } - ishm_tbl->block[block_index].fragment = fragment; + new_block->fragment = fragment; } /* try to mmap: */ @@ -771,6 +768,65 @@ static void procsync(void) } /* + * Free a block as described in block_free(), but + * considering whether to close the file descriptor or not, and + * whether to deregister from the fdserver. + */ +static int block_free_internal(int block_index, int close_fd, int deregister) +{ + int proc_index; + ishm_block_t *block; /* entry in the main block table*/ + int last; + int ret = 0; + + if ((block_index < 0) || + (block_index >= ISHM_MAX_NB_BLOCKS) || + (ishm_tbl->block[block_index].len == 0)) { + ODP_ERR("Request to free an invalid block\n"); + return -1; + } + + block = &ishm_tbl->block[block_index]; + + proc_index = procfind_block(block_index); + if (proc_index >= 0) { + /* remove the mapping and possible fragment */ + do_unmap(ishm_proctable->entry[proc_index].start, + block->len, + ishm_proctable->entry[proc_index].flags, + block_index); + + /* close the related fd */ + if (close_fd) + close(ishm_proctable->entry[proc_index].fd); + + /* remove entry from process local table: */ + last = ishm_proctable->nb_entries - 1; + ishm_proctable->entry[proc_index] = ishm_proctable->entry[last]; + ishm_proctable->nb_entries = last; + } else { + /* just possibly free the fragment as no mapping exist here: */ + do_unmap(NULL, 0, block->flags, block_index); + } + + /* remove all files related to this block: */ + if (close_fd) + delete_file(block); + + /* deregister the file descriptor from the file descriptor server. */ + if (deregister) + ret = _odp_fdserver_deregister_fd(FD_SRV_CTX_ISHM, block_index); + + /* mark the block as free in the main block table: */ + block->len = 0; + + /* mark the change so other processes see this entry as obsolete: */ + block->seq++; + + return ret; +} + +/* * Allocate and map internal shared memory, or other objects: * If a name is given, check that this name is not already in use. * If ok, allocate a new shared memory block and map the @@ -834,7 +890,6 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, /* If a file descriptor is provided, get the real size and map: */ if (fd >= 0) { if (fstat(fd, &statbuf) < 0) { - close(fd); odp_spinlock_unlock(&ishm_tbl->lock); ODP_ERR("_ishm_reserve failed (fstat failed: %s).\n", strerror(errno)); @@ -842,17 +897,16 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, return -1; } len = statbuf.st_size; + new_block->external_fd = 1; /* note that the huge page flag is meningless here as huge * page is determined by the provided file descriptor: */ addr = do_map(new_index, len, align, flags, EXTERNAL, &fd); if (addr == NULL) { - close(fd); odp_spinlock_unlock(&ishm_tbl->lock); ODP_ERR("_ishm_reserve failed.\n"); return -1; } new_block->huge = EXTERNAL; - new_block->external_fd = 1; } else { new_block->external_fd = 0; } @@ -933,7 +987,10 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, ishm_proctable->entry[new_proc_entry].fd = fd; /* register the file descriptor to the file descriptor server. */ - _odp_fdserver_register_fd(FD_SRV_CTX_ISHM, new_index, fd); + if (_odp_fdserver_register_fd(FD_SRV_CTX_ISHM, new_index, fd) == -1) { + block_free_internal(new_index, !new_block->external_fd, 0); + new_index = -1; + } odp_spinlock_unlock(&ishm_tbl->lock); return new_index; @@ -1039,53 +1096,7 @@ error_exp_file: */ static int block_free(int block_index) { - int proc_index; - ishm_block_t *block; /* entry in the main block table*/ - int last; - - if ((block_index < 0) || - (block_index >= ISHM_MAX_NB_BLOCKS) || - (ishm_tbl->block[block_index].len == 0)) { - ODP_ERR("Request to free an invalid block\n"); - return -1; - } - - block = &ishm_tbl->block[block_index]; - - proc_index = procfind_block(block_index); - if (proc_index >= 0) { - /* close the related fd */ - close(ishm_proctable->entry[proc_index].fd); - - /* remove the mapping and possible fragment */ - do_unmap(ishm_proctable->entry[proc_index].start, - block->len, - ishm_proctable->entry[proc_index].flags, - block_index); - - /* remove entry from process local table: */ - last = ishm_proctable->nb_entries - 1; - ishm_proctable->entry[proc_index] = - ishm_proctable->entry[last]; - ishm_proctable->nb_entries = last; - } else { - /* just possibly free the fragment as no mapping exist here: */ - do_unmap(NULL, 0, block->flags, block_index); - } - - /* remove all files related to this block: */ - delete_file(block); - - /* deregister the file descriptor from the file descriptor server. */ - _odp_fdserver_deregister_fd(FD_SRV_CTX_ISHM, block_index); - - /* mark the block as free in the main block table: */ - block->len = 0; - - /* mark the change so other processes see this entry as obsolete: */ - block->seq++; - - return 0; + return block_free_internal(block_index, 1, 1); } /* diff --git a/platform/linux-generic/_ishmphy.c b/platform/linux-generic/odp_ishmphy.c index 8dde2831e..6207ce757 100644 --- a/platform/linux-generic/_ishmphy.c +++ b/platform/linux-generic/odp_ishmphy.c @@ -18,8 +18,8 @@ #include <odp/api/debug.h> #include <odp_debug_internal.h> #include <odp_align_internal.h> -#include <_ishm_internal.h> -#include <_ishmphy_internal.h> +#include <odp_ishm_internal.h> +#include <odp_ishmphy_internal.h> #include <stdlib.h> #include <stdio.h> @@ -31,7 +31,7 @@ #include <fcntl.h> #include <sys/types.h> #include <sys/wait.h> -#include <_ishmphy_internal.h> +#include <odp_ishmphy_internal.h> static void *common_va_address; static uint64_t common_va_len; diff --git a/platform/linux-generic/_ishmpool.c b/platform/linux-generic/odp_ishmpool.c index 4ff24c026..04a0e535a 100644 --- a/platform/linux-generic/_ishmpool.c +++ b/platform/linux-generic/odp_ishmpool.c @@ -51,8 +51,8 @@ #include <odp_shm_internal.h> #include <odp_debug_internal.h> #include <odp_align_internal.h> -#include <_ishm_internal.h> -#include <_ishmpool_internal.h> +#include <odp_ishm_internal.h> +#include <odp_ishmpool_internal.h> #include <stdlib.h> #include <stdio.h> #include <unistd.h> diff --git a/platform/linux-generic/odp_libconfig.c b/platform/linux-generic/odp_libconfig.c index 3b3b31703..316ed067a 100644 --- a/platform/linux-generic/odp_libconfig.c +++ b/platform/linux-generic/odp_libconfig.c @@ -16,8 +16,6 @@ #include <odp_libconfig_internal.h> #include <odp_libconfig_config.h> -#define CONF_STR_NAME ((const char *)odp_linux_generic_conf) - extern struct odp_global_data_s odp_global_data; int _odp_libconfig_init_global(void) @@ -33,7 +31,7 @@ int _odp_libconfig_init_global(void) config_init(config); config_init(config_rt); - if (!config_read_string(config, CONF_STR_NAME)) { + if (!config_read_string(config, config_builtin)) { ODP_ERR("Failed to read default config: %s(%d): %s\n", config_error_file(config), config_error_line(config), config_error_text(config)); @@ -97,3 +95,41 @@ int _odp_libconfig_lookup_int(const char *path, int *value) return (ret_def == CONFIG_TRUE || ret_rt == CONFIG_TRUE) ? 1 : 0; } + +static int lookup_int(config_t *cfg, + const char *base_path, + const char *local_path, + const char *name, + int *value) +{ + char path[256]; + + if (local_path) { + snprintf(path, sizeof(path), "%s.%s.%s", base_path, + local_path, name); + if (config_lookup_int(cfg, path, value) == CONFIG_TRUE) + return 1; + } + + snprintf(path, sizeof(path), "%s.%s", base_path, name); + if (config_lookup_int(cfg, path, value) == CONFIG_TRUE) + return 1; + + return 0; +} + +int _odp_libconfig_lookup_ext_int(const char *base_path, + const char *local_path, + const char *name, + int *value) +{ + if (lookup_int(&odp_global_data.libconfig_runtime, + base_path, local_path, name, value)) + return 1; + + if (lookup_int(&odp_global_data.libconfig_default, + base_path, local_path, name, value)) + return 1; + + return 0; +} diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 462c8a4c7..d4b19feeb 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1910,6 +1910,92 @@ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt) return dst_uarea_size < src_uarea_size; } +/* Simple implementation of ones complement sum. + * Based on RFC1071 and its errata. + */ +typedef union { + uint16_t w; + uint8_t b[2]; +} swap_buf_t; + +static uint32_t segment_sum16_32(const uint8_t *p, + uint32_t len, + uint32_t offset) + +{ + uint32_t sum = 0; + + /* Include second part of 16-bit short word split between segments */ + if (len > 0 && (offset % 2)) { + swap_buf_t sw; + + sw.b[0] = 0; + sw.b[1] = *p++; + sum = sw.w; + len--; + } + + /* + * If pointer is 16-bit aligned, we can do fast path calculation. + * If it is not, we sum hi and lo bytes separately and then sum them. + */ + if ((uintptr_t)p % 2) { + uint32_t sum1 = 0, sum2 = 0; + + while (len > 1) { + sum1 += *p++; + sum2 += *p++; + len -= 2; + } +#if (ODP_BYTE_ORDER == ODP_BIG_ENDIAN) + sum += sum2 + (sum1 << 8); +#else + sum += sum1 + (sum2 << 8); +#endif + } else { + while (len > 1) { + sum += *(const uint16_t *)(uintptr_t)p; + p += 2; + len -= 2; + } + } + + /* Add left-over byte, if any */ + if (len > 0) { + swap_buf_t sw; + + sw.b[0] = *p; + sw.b[1] = 0; + sum += sw.w; + } + + return sum; +} + +static uint32_t packet_sum16_32(odp_packet_hdr_t *pkt_hdr, + uint32_t offset, + uint32_t len) +{ + uint32_t sum = 0; + + if (offset + len > pkt_hdr->frame_len) + return 0; + + while (len > 0) { + uint32_t seglen = 0; /* GCC */ + void *mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); + + if (seglen > len) + seglen = len; + + sum += segment_sum16_32(mapaddr, seglen, offset); + len -= seglen; + offset += seglen; + } + + return sum; +} + /** Parser helper function for Ethernet packets */ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len) @@ -2266,6 +2352,159 @@ int packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, seg_len, layer, ethtype); } +static inline int packet_ipv4_chksum(odp_packet_t pkt, + uint32_t offset, + _odp_ipv4hdr_t *ip, + odp_u16sum_t *chksum) +{ + unsigned int nleft = _ODP_IPV4HDR_IHL(ip->ver_ihl) * 4; + uint16_t buf[nleft / 2]; + int res; + + if (odp_unlikely(nleft < sizeof(*ip))) + return -1; + ip->chksum = 0; + memcpy(buf, ip, sizeof(*ip)); + res = odp_packet_copy_to_mem(pkt, offset + sizeof(*ip), + nleft - sizeof(*ip), + buf + sizeof(*ip) / 2); + if (odp_unlikely(res < 0)) + return res; + + *chksum = ~odp_chksum_ones_comp16(buf, nleft); + + return 0; +} + +#define _ODP_IPV4HDR_CSUM_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, chksum) +#define _ODP_IPV4ADDR_OFFSSET ODP_OFFSETOF(_odp_ipv4hdr_t, src_addr) +#define _ODP_IPV6ADDR_OFFSSET ODP_OFFSETOF(_odp_ipv6hdr_t, src_addr) +#define _ODP_IPV4HDR_CSUM_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, chksum) +#define _ODP_UDP_LEN_OFFSET ODP_OFFSETOF(_odp_udphdr_t, length) +#define _ODP_UDP_CSUM_OFFSET ODP_OFFSETOF(_odp_udphdr_t, chksum) + +/** + * Calculate and fill in IPv4 checksum + * + * @param pkt ODP packet + * + * @retval 0 on success + * @retval <0 on failure + */ +int _odp_packet_ipv4_chksum_insert(odp_packet_t pkt) +{ + uint32_t offset; + _odp_ipv4hdr_t ip; + odp_u16sum_t chksum; + int res; + + offset = odp_packet_l3_offset(pkt); + if (offset == ODP_PACKET_OFFSET_INVALID) + return -1; + + res = odp_packet_copy_to_mem(pkt, offset, sizeof(ip), &ip); + if (odp_unlikely(res < 0)) + return res; + + res = packet_ipv4_chksum(pkt, offset, &ip, &chksum); + if (odp_unlikely(res < 0)) + return res; + + return odp_packet_copy_from_mem(pkt, + offset + _ODP_IPV4HDR_CSUM_OFFSET, + 2, &chksum); +} + +static int _odp_packet_tcp_udp_chksum_insert(odp_packet_t pkt, uint16_t proto) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t zero = 0; + uint32_t sum; + uint16_t l3_ver; + uint16_t chksum; + uint32_t chksum_offset; + + if (pkt_hdr->p.l3_offset == ODP_PACKET_OFFSET_INVALID) + return -1; + if (pkt_hdr->p.l4_offset == ODP_PACKET_OFFSET_INVALID) + return -1; + + odp_packet_copy_to_mem(pkt, pkt_hdr->p.l3_offset, 2, &l3_ver); + + if (_ODP_IPV4HDR_VER(l3_ver) == _ODP_IPV4) + sum = packet_sum16_32(pkt_hdr, + pkt_hdr->p.l3_offset + + _ODP_IPV4ADDR_OFFSSET, + 2 * _ODP_IPV4ADDR_LEN); + else + sum = packet_sum16_32(pkt_hdr, + pkt_hdr->p.l3_offset + + _ODP_IPV6ADDR_OFFSSET, + 2 * _ODP_IPV6ADDR_LEN); +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN + sum += proto; +#else + sum += proto << 8; +#endif + + if (proto == _ODP_IPPROTO_TCP) { + sum += _odp_cpu_to_be_16(pkt_hdr->frame_len - + pkt_hdr->p.l4_offset); + chksum_offset = pkt_hdr->p.l4_offset + _ODP_UDP_CSUM_OFFSET; + } else { + sum += packet_sum16_32(pkt_hdr, + pkt_hdr->p.l4_offset + + _ODP_UDP_LEN_OFFSET, + 2); + chksum_offset = pkt_hdr->p.l4_offset + _ODP_UDP_CSUM_OFFSET; + } + odp_packet_copy_from_mem(pkt, chksum_offset, 2, &zero); + + sum += packet_sum16_32(pkt_hdr, + pkt_hdr->p.l4_offset, + pkt_hdr->frame_len - + pkt_hdr->p.l4_offset); + + /* Not more than two additions */ + sum = (sum & 0xffff) + (sum >> 16); + sum = (sum & 0xffff) + (sum >> 16); + + chksum = ~sum; + + if (proto == _ODP_IPPROTO_UDP && chksum == 0) + chksum = 0xffff; + + return odp_packet_copy_from_mem(pkt, + chksum_offset, + 2, &chksum); +} + +/** + * Calculate and fill in TCP checksum + * + * @param pkt ODP packet + * + * @retval 0 on success + * @retval <0 on failure + */ +int _odp_packet_tcp_chksum_insert(odp_packet_t pkt) +{ + return _odp_packet_tcp_udp_chksum_insert(pkt, _ODP_IPPROTO_TCP); +} + +/** + * Calculate and fill in UDP checksum + * + * @param pkt ODP packet + * + * @retval 0 on success + * @retval <0 on failure + */ +int _odp_packet_udp_chksum_insert(odp_packet_t pkt) +{ + return _odp_packet_tcp_udp_chksum_insert(pkt, _ODP_IPPROTO_UDP); +} + /** * Simple packet parser */ diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 56e9f6860..bf7aef148 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -376,8 +376,10 @@ int odp_pktio_close(odp_pktio_t hdl) int res; entry = get_pktio_entry(hdl); - if (entry == NULL) + if (entry == NULL) { + ODP_ERR("Bad handle\n"); return -1; + } if (entry->s.state == PKTIO_STATE_STARTED) { ODP_DBG("Missing odp_pktio_stop() before close.\n"); @@ -416,8 +418,10 @@ int odp_pktio_config(odp_pktio_t hdl, const odp_pktio_config_t *config) int res = 0; entry = get_pktio_entry(hdl); - if (!entry) + if (!entry) { + ODP_ERR("Bad handle\n"); return -1; + } if (config == NULL) { odp_pktio_config_init(&default_config); @@ -466,12 +470,15 @@ int odp_pktio_start(odp_pktio_t hdl) int res = 0; entry = get_pktio_entry(hdl); - if (!entry) + if (!entry) { + ODP_ERR("Bad handle\n"); return -1; + } lock_entry(entry); if (entry->s.state == PKTIO_STATE_STARTED) { unlock_entry(entry); + ODP_ERR("Already started\n"); return -1; } if (entry->s.ops->start) @@ -513,8 +520,10 @@ static int _pktio_stop(pktio_entry_t *entry) int res = 0; odp_pktin_mode_t mode = entry->s.param.in_mode; - if (entry->s.state != PKTIO_STATE_STARTED) + if (entry->s.state != PKTIO_STATE_STARTED) { + ODP_ERR("Not started\n"); return -1; + } if (entry->s.ops->stop) res = entry->s.ops->stop(entry); @@ -536,8 +545,10 @@ int odp_pktio_stop(odp_pktio_t hdl) int res; entry = get_pktio_entry(hdl); - if (!entry) + if (!entry) { + ODP_ERR("Bad handle\n"); return -1; + } lock_entry(entry); res = _pktio_stop(entry); @@ -1159,13 +1170,15 @@ void odp_pktio_print(odp_pktio_t hdl) len += snprintf(&str[len], n - len, "pktio\n"); len += snprintf(&str[len], n - len, - " handle %" PRIu64 "\n", - odp_pktio_to_u64(hdl)); - len += snprintf(&str[len], n - len, " name %s\n", entry->s.name); len += snprintf(&str[len], n - len, " type %s\n", entry->s.ops->name); len += snprintf(&str[len], n - len, + " index %i\n", _odp_pktio_index(hdl)); + len += snprintf(&str[len], n - len, + " handle (u64) %" PRIu64 "\n", + odp_pktio_to_u64(hdl)); + len += snprintf(&str[len], n - len, " state %s\n", entry->s.state == PKTIO_STATE_STARTED ? "start" : (entry->s.state == PKTIO_STATE_STOPPED ? "stop" : diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 998fc649e..955e14f9b 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -718,12 +718,13 @@ int buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int max_num) buf_hdr[i] = buf_hdr_from_index(pool, cache->buf_index[j]); } + /* Declare variable here to fix clang compilation bug */ + uint32_t data[burst]; + /* If needed, get more from the global pool */ if (odp_unlikely(num_deq)) { - /* Temporary copy needed since odp_buffer_t is uintptr_t - * and not uint32_t. */ - uint32_t data[burst]; - + /* Temporary copy to data[] needed since odp_buffer_t is + * uintptr_t and not uint32_t. */ ring = &pool->ring->hdr; mask = pool->ring_mask; burst = ring_deq_multi(ring, mask, data, burst); diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 1218987b1..89a0cd907 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -25,6 +25,7 @@ #include <odp/api/hints.h> #include <odp/api/sync.h> #include <odp/api/traffic_mngr.h> +#include <odp_libconfig_internal.h> #define NUM_INTERNAL_QUEUES 64 @@ -36,6 +37,9 @@ #include <string.h> #include <inttypes.h> +#define MIN_QUEUE_SIZE 8 +#define MAX_QUEUE_SIZE (1 * 1024 * 1024) + static int queue_init(queue_entry_t *queue, const char *name, const odp_queue_param_t *param); @@ -60,11 +64,11 @@ static int queue_capa(odp_queue_capability_t *capa, int sched) /* Reserve some queues for internal use */ capa->max_queues = ODP_CONFIG_QUEUES - NUM_INTERNAL_QUEUES; capa->plain.max_num = capa->max_queues; - capa->plain.max_size = CONFIG_QUEUE_SIZE; + capa->plain.max_size = queue_glb->config.max_queue_size; capa->plain.lockfree.max_num = queue_glb->queue_lf_num; capa->plain.lockfree.max_size = queue_glb->queue_lf_size; capa->sched.max_num = capa->max_queues; - capa->sched.max_size = CONFIG_QUEUE_SIZE; + capa->sched.max_size = queue_glb->config.max_queue_size; if (sched) { capa->max_ordered_locks = sched_fn->max_ordered_locks(); @@ -75,6 +79,52 @@ static int queue_capa(odp_queue_capability_t *capa, int sched) return 0; } +static int read_config_file(queue_global_t *queue_glb) +{ + const char *str; + uint32_t val_u32; + int val = 0; + + ODP_PRINT("Queue config:\n"); + + str = "queue_basic.max_queue_size"; + if (!_odp_libconfig_lookup_int(str, &val)) { + ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + + val_u32 = val; + + if (val_u32 > MAX_QUEUE_SIZE || val_u32 < MIN_QUEUE_SIZE || + !CHECK_IS_POWER2(val_u32)) { + ODP_ERR("Bad value %s = %u\n", str, val_u32); + return -1; + } + + queue_glb->config.max_queue_size = val_u32; + ODP_PRINT(" %s: %u\n", str, val_u32); + + str = "queue_basic.default_queue_size"; + if (!_odp_libconfig_lookup_int(str, &val)) { + ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + + val_u32 = val; + + if (val_u32 > queue_glb->config.max_queue_size || + val_u32 < MIN_QUEUE_SIZE || + !CHECK_IS_POWER2(val_u32)) { + ODP_ERR("Bad value %s = %u\n", str, val_u32); + return -1; + } + + queue_glb->config.default_queue_size = val_u32; + ODP_PRINT(" %s: %u\n\n", str, val_u32); + + return 0; +} + static int queue_init_global(void) { uint32_t i; @@ -82,10 +132,11 @@ static int queue_init_global(void) uint32_t lf_size = 0; queue_lf_func_t *lf_func; odp_queue_capability_t capa; + uint64_t mem_size; ODP_DBG("Starts...\n"); - shm = odp_shm_reserve("odp_queues", + shm = odp_shm_reserve("_odp_queue_gbl", sizeof(queue_global_t), sizeof(queue_entry_t), 0); @@ -104,6 +155,26 @@ static int queue_init_global(void) queue->s.handle = queue_from_index(i); } + if (read_config_file(queue_glb)) { + odp_shm_free(shm); + return -1; + } + + queue_glb->queue_gbl_shm = shm; + mem_size = sizeof(uint32_t) * ODP_CONFIG_QUEUES * + (uint64_t)queue_glb->config.max_queue_size; + + shm = odp_shm_reserve("_odp_queue_rings", mem_size, + ODP_CACHE_LINE_SIZE, 0); + + if (shm == ODP_SHM_INVALID) { + odp_shm_free(queue_glb->queue_gbl_shm); + return -1; + } + + queue_glb->queue_ring_shm = shm; + queue_glb->ring_data = odp_shm_addr(shm); + lf_func = &queue_glb->queue_lf_func; queue_glb->queue_lf_num = queue_lf_init_global(&lf_size, lf_func); queue_glb->queue_lf_size = lf_size; @@ -133,7 +204,6 @@ static int queue_term_local(void) static int queue_term_global(void) { int ret = 0; - int rc = 0; queue_entry_t *queue; int i; @@ -142,20 +212,24 @@ static int queue_term_global(void) LOCK(queue); if (queue->s.status != QUEUE_STATUS_FREE) { ODP_ERR("Not destroyed queue: %s\n", queue->s.name); - rc = -1; + ret = -1; } UNLOCK(queue); } queue_lf_term_global(); - ret = odp_shm_free(odp_shm_lookup("odp_queues")); - if (ret < 0) { - ODP_ERR("shm free failed for odp_queues"); - rc = -1; + if (odp_shm_free(queue_glb->queue_ring_shm)) { + ODP_ERR("shm free failed"); + ret = -1; } - return rc; + if (odp_shm_free(queue_glb->queue_gbl_shm)) { + ODP_ERR("shm free failed"); + ret = -1; + } + + return ret; } static int queue_capability(odp_queue_capability_t *capa) @@ -207,7 +281,7 @@ static odp_queue_t queue_create(const char *name, } if (param->nonblocking == ODP_BLOCKING) { - if (param->size > CONFIG_QUEUE_SIZE) + if (param->size > queue_glb->config.max_queue_size) return ODP_QUEUE_INVALID; } else if (param->nonblocking == ODP_NONBLOCKING_LF) { /* Only plain type lock-free queues supported */ @@ -586,6 +660,9 @@ static odp_event_t queue_deq(odp_queue_t handle) static int queue_init(queue_entry_t *queue, const char *name, const odp_queue_param_t *param) { + uint64_t offset; + uint32_t queue_size; + if (name == NULL) { queue->s.name[0] = 0; } else { @@ -609,9 +686,24 @@ static int queue_init(queue_entry_t *queue, const char *name, queue->s.pktin = PKTIN_INVALID; queue->s.pktout = PKTOUT_INVALID; - ring_st_init(&queue->s.ring_st, - queue_glb->ring_data[queue->s.index].data, - CONFIG_QUEUE_SIZE); + /* Use default size for all small queues to quarantee performance + * level. */ + queue_size = queue_glb->config.default_queue_size; + if (param->size > queue_glb->config.default_queue_size) + queue_size = param->size; + + /* Round up if not already a power of two */ + queue_size = ROUNDUP_POWER2_U32(queue_size); + + if (queue_size > queue_glb->config.max_queue_size) { + ODP_ERR("Too large queue size %u\n", queue_size); + return -1; + } + + offset = queue->s.index * (uint64_t)queue_glb->config.max_queue_size; + + ring_st_init(&queue->s.ring_st, &queue_glb->ring_data[offset], + queue_size); return 0; } diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index 895133cd5..2a5e01d47 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -25,8 +25,8 @@ #include <odp_pool_internal.h> #include <odp_queue_scalable_internal.h> #include <odp_schedule_if.h> -#include <_ishm_internal.h> -#include <_ishmpool_internal.h> +#include <odp_ishm_internal.h> +#include <odp_ishmpool_internal.h> #include <string.h> #include <inttypes.h> diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index cd20b39dd..b50462a55 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -27,6 +27,7 @@ #include <odp_timer_internal.h> #include <odp_queue_internal.h> #include <odp_buffer_inlines.h> +#include <odp_libconfig_internal.h> /* Number of priority levels */ #define NUM_PRIO 8 @@ -41,15 +42,21 @@ ODP_STATIC_ASSERT((ODP_SCHED_PRIO_NORMAL > 0) && /* Number of scheduling groups */ #define NUM_SCHED_GRPS 32 -/* Priority queues per priority */ -#define QUEUES_PER_PRIO 4 +/* Group weight table size */ +#define GRP_WEIGHT_TBL_SIZE NUM_SCHED_GRPS + +/* Maximum priority queue spread */ +#define MAX_SPREAD 8 + +/* Minimum priority queue spread */ +#define MIN_SPREAD 1 /* A thread polls a non preferred sched queue every this many polls * of the prefer queue. */ #define PREFER_RATIO 64 -/* Size of poll weight table */ -#define WEIGHT_TBL_SIZE ((QUEUES_PER_PRIO - 1) * PREFER_RATIO) +/* Spread weight table */ +#define SPREAD_TBL_SIZE ((MAX_SPREAD - 1) * PREFER_RATIO) /* Maximum number of packet IO interfaces */ #define NUM_PKTIO ODP_CONFIG_PKTIO_ENTRIES @@ -60,14 +67,10 @@ ODP_STATIC_ASSERT((ODP_SCHED_PRIO_NORMAL > 0) && /* Not a valid index */ #define NULL_INDEX ((uint32_t)-1) -/* Priority queue ring size. In worst case, all event queues are scheduled - * queues and have the same priority. The ring size must be larger than or - * equal to ODP_CONFIG_QUEUES / QUEUES_PER_PRIO, so that it can hold all - * queues in the worst case. */ -#define PRIO_QUEUE_RING_SIZE (ODP_CONFIG_QUEUES / QUEUES_PER_PRIO) - -/* Mask for wrapping around priority queue index */ -#define RING_MASK (PRIO_QUEUE_RING_SIZE - 1) +/* Maximum priority queue ring size. A ring must be large enough to store all + * queues in the worst case (all queues are scheduled, have the same priority + * and no spreading). */ +#define MAX_RING_SIZE ODP_CONFIG_QUEUES /* Priority queue empty, not a valid queue index. */ #define PRIO_QUEUE_EMPTY NULL_INDEX @@ -76,14 +79,14 @@ ODP_STATIC_ASSERT((ODP_SCHED_PRIO_NORMAL > 0) && ODP_STATIC_ASSERT(CHECK_IS_POWER2(ODP_CONFIG_QUEUES), "Number_of_queues_is_not_power_of_two"); -/* Ring size must be power of two, so that MAX_QUEUE_IDX_MASK can be used. */ -ODP_STATIC_ASSERT(CHECK_IS_POWER2(PRIO_QUEUE_RING_SIZE), +/* Ring size must be power of two, so that mask can be used. */ +ODP_STATIC_ASSERT(CHECK_IS_POWER2(MAX_RING_SIZE), "Ring_size_is_not_power_of_two"); /* Mask of queues per priority */ typedef uint8_t pri_mask_t; -ODP_STATIC_ASSERT((8 * sizeof(pri_mask_t)) >= QUEUES_PER_PRIO, +ODP_STATIC_ASSERT((8 * sizeof(pri_mask_t)) >= MAX_SPREAD, "pri_mask_t_is_too_small"); /* Start of named groups in group mask arrays */ @@ -116,17 +119,18 @@ typedef struct { int thr; uint16_t stash_num; uint16_t stash_index; - uint16_t pause; - uint16_t round; + uint16_t grp_round; + uint16_t spread_round; uint32_t stash_qi; odp_queue_t stash_queue; odp_event_t stash_ev[MAX_DEQ]; uint32_t grp_epoch; - int num_grp; + uint16_t num_grp; + uint16_t pause; uint8_t grp[NUM_SCHED_GRPS]; - uint8_t weight_tbl[WEIGHT_TBL_SIZE]; - uint8_t grp_weight[WEIGHT_TBL_SIZE]; + uint8_t spread_tbl[SPREAD_TBL_SIZE]; + uint8_t grp_weight[GRP_WEIGHT_TBL_SIZE]; struct { /* Source queue index */ @@ -147,7 +151,7 @@ typedef struct ODP_ALIGNED_CACHE { ring_t ring; /* Ring data: queue indexes */ - uint32_t queue_index[PRIO_QUEUE_RING_SIZE]; + uint32_t queue_index[MAX_RING_SIZE]; } prio_queue_t; @@ -168,14 +172,21 @@ typedef struct { pri_mask_t pri_mask[NUM_PRIO]; odp_spinlock_t mask_lock; - prio_queue_t prio_q[NUM_SCHED_GRPS][NUM_PRIO][QUEUES_PER_PRIO]; + prio_queue_t prio_q[NUM_SCHED_GRPS][NUM_PRIO][MAX_SPREAD]; odp_shm_t shm; - uint32_t pri_count[NUM_PRIO][QUEUES_PER_PRIO]; + + struct { + uint8_t num_spread; + } config; + + uint32_t pri_count[NUM_PRIO][MAX_SPREAD]; odp_thrmask_t mask_all; odp_spinlock_t grp_lock; odp_atomic_u32_t grp_epoch; + uint32_t ring_mask; + uint16_t max_spread; struct { char name[ODP_SCHED_GROUP_NAME_LEN]; @@ -186,7 +197,7 @@ typedef struct { struct { uint8_t grp; uint8_t prio; - uint8_t queue_per_prio; + uint8_t spread; uint8_t sync; uint8_t order_lock_count; uint8_t poll_pktin; @@ -206,11 +217,11 @@ typedef struct { /* Check that queue[] variables are large enough */ ODP_STATIC_ASSERT(NUM_SCHED_GRPS <= 256, "Group_does_not_fit_8_bits"); ODP_STATIC_ASSERT(NUM_PRIO <= 256, "Prio_does_not_fit_8_bits"); -ODP_STATIC_ASSERT(QUEUES_PER_PRIO <= 256, - "Queues_per_prio_does_not_fit_8_bits"); +ODP_STATIC_ASSERT(MAX_SPREAD <= 256, "Spread_does_not_fit_8_bits"); ODP_STATIC_ASSERT(CONFIG_QUEUE_MAX_ORD_LOCKS <= 256, "Ordered_lock_count_does_not_fit_8_bits"); ODP_STATIC_ASSERT(NUM_PKTIO <= 256, "Pktio_index_does_not_fit_8_bits"); +ODP_STATIC_ASSERT(CHECK_IS_POWER2(GRP_WEIGHT_TBL_SIZE), "Not_power_of_2"); /* Global scheduler context */ static sched_global_t *sched; @@ -221,11 +232,41 @@ static __thread sched_local_t sched_local; /* Function prototypes */ static inline void schedule_release_context(void); +static int read_config_file(sched_global_t *sched) +{ + const char *str; + int val = 0; + + ODP_PRINT("Scheduler config:\n"); + + str = "sched_basic.prio_spread"; + if (!_odp_libconfig_lookup_int(str, &val)) { + ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + + if (val > MAX_SPREAD || val < MIN_SPREAD) { + ODP_ERR("Bad value %s = %u\n", str, val); + return -1; + } + + sched->config.num_spread = val; + ODP_PRINT(" %s: %i\n\n", str, val); + + return 0; +} + +static inline uint8_t prio_spread_index(uint32_t index) +{ + return index % sched->config.num_spread; +} + static void sched_local_init(void) { int i; - uint8_t id; - uint8_t offset = 0; + uint8_t spread; + uint8_t num_spread = sched->config.num_spread; + uint8_t offset = 1; memset(&sched_local, 0, sizeof(sched_local_t)); @@ -234,17 +275,17 @@ static void sched_local_init(void) sched_local.stash_qi = PRIO_QUEUE_EMPTY; sched_local.ordered.src_queue = NULL_INDEX; - id = sched_local.thr & (QUEUES_PER_PRIO - 1); + spread = prio_spread_index(sched_local.thr); - for (i = 0; i < WEIGHT_TBL_SIZE; i++) { - sched_local.weight_tbl[i] = id; + for (i = 0; i < SPREAD_TBL_SIZE; i++) { + sched_local.spread_tbl[i] = spread; - if (i % PREFER_RATIO == 0) { + if (num_spread > 1 && (i % PREFER_RATIO) == 0) { + sched_local.spread_tbl[i] = prio_spread_index(spread + + offset); offset++; - sched_local.weight_tbl[i] = (id + offset) & - (QUEUES_PER_PRIO - 1); - if (offset == QUEUES_PER_PRIO - 1) - offset = 0; + if (offset == num_spread) + offset = 1; } } } @@ -269,19 +310,26 @@ static int schedule_init_global(void) memset(sched, 0, sizeof(sched_global_t)); + if (read_config_file(sched)) { + odp_shm_free(shm); + return -1; + } + + /* When num_spread == 1, only spread_tbl[0] is used. */ + sched->max_spread = (sched->config.num_spread - 1) * PREFER_RATIO; sched->shm = shm; odp_spinlock_init(&sched->mask_lock); for (grp = 0; grp < NUM_SCHED_GRPS; grp++) { for (i = 0; i < NUM_PRIO; i++) { - for (j = 0; j < QUEUES_PER_PRIO; j++) { + for (j = 0; j < MAX_SPREAD; j++) { prio_queue_t *prio_q; int k; prio_q = &sched->prio_q[grp][i][j]; ring_init(&prio_q->ring); - for (k = 0; k < PRIO_QUEUE_RING_SIZE; k++) { + for (k = 0; k < MAX_RING_SIZE; k++) { prio_q->queue_index[k] = PRIO_QUEUE_EMPTY; } @@ -322,14 +370,15 @@ static int schedule_term_global(void) int ret = 0; int rc = 0; int i, j, grp; + uint32_t ring_mask = sched->ring_mask; for (grp = 0; grp < NUM_SCHED_GRPS; grp++) { for (i = 0; i < NUM_PRIO; i++) { - for (j = 0; j < QUEUES_PER_PRIO; j++) { + for (j = 0; j < MAX_SPREAD; j++) { ring_t *ring = &sched->prio_q[grp][i][j].ring; uint32_t qi; - while ((qi = ring_deq(ring, RING_MASK)) != + while ((qi = ring_deq(ring, ring_mask)) != RING_EMPTY) { odp_event_t events[1]; int num; @@ -401,7 +450,7 @@ static inline int grp_update_tbl(void) odp_spinlock_unlock(&sched->grp_lock); /* Update group weights. Round robin over all thread's groups. */ - for (i = 0; i < WEIGHT_TBL_SIZE; i++) + for (i = 0; i < GRP_WEIGHT_TBL_SIZE; i++) sched_local.grp_weight[i] = i % num; sched_local.num_grp = num; @@ -413,11 +462,6 @@ static uint32_t schedule_max_ordered_locks(void) return CONFIG_QUEUE_MAX_ORD_LOCKS; } -static inline int queue_per_prio(uint32_t queue_index) -{ - return ((QUEUES_PER_PRIO - 1) & queue_index); -} - static void pri_set(int id, int prio) { odp_spinlock_lock(&sched->mask_lock); @@ -441,33 +485,39 @@ static void pri_clr(int id, int prio) static void pri_set_queue(uint32_t queue_index, int prio) { - int id = queue_per_prio(queue_index); + uint8_t id = prio_spread_index(queue_index); return pri_set(id, prio); } static void pri_clr_queue(uint32_t queue_index, int prio) { - int id = queue_per_prio(queue_index); + uint8_t id = prio_spread_index(queue_index); pri_clr(id, prio); } static int schedule_init_queue(uint32_t queue_index, const odp_schedule_param_t *sched_param) { + uint32_t ring_size; int i; int prio = sched_param->prio; pri_set_queue(queue_index, prio); sched->queue[queue_index].grp = sched_param->group; sched->queue[queue_index].prio = prio; - sched->queue[queue_index].queue_per_prio = queue_per_prio(queue_index); + sched->queue[queue_index].spread = prio_spread_index(queue_index); sched->queue[queue_index].sync = sched_param->sync; sched->queue[queue_index].order_lock_count = sched_param->lock_count; sched->queue[queue_index].poll_pktin = 0; sched->queue[queue_index].pktio_index = 0; sched->queue[queue_index].pktin_index = 0; + ring_size = MAX_RING_SIZE / sched->config.num_spread; + ring_size = ROUNDUP_POWER2_U32(ring_size); + ODP_ASSERT(ring_size <= MAX_RING_SIZE); + sched->ring_mask = ring_size - 1; + odp_atomic_init_u64(&sched->order[queue_index].ctx, 0); odp_atomic_init_u64(&sched->order[queue_index].next_ctx, 0); @@ -492,9 +542,9 @@ static void schedule_destroy_queue(uint32_t queue_index) int prio = sched->queue[queue_index].prio; pri_clr_queue(queue_index, prio); - sched->queue[queue_index].grp = 0; - sched->queue[queue_index].prio = 0; - sched->queue[queue_index].queue_per_prio = 0; + sched->queue[queue_index].grp = 0; + sched->queue[queue_index].prio = 0; + sched->queue[queue_index].spread = 0; if (queue_is_ordered(queue_index) && odp_atomic_load_u64(&sched->order[queue_index].ctx) != @@ -504,12 +554,12 @@ static void schedule_destroy_queue(uint32_t queue_index) static int schedule_sched_queue(uint32_t queue_index) { - int grp = sched->queue[queue_index].grp; - int prio = sched->queue[queue_index].prio; - int queue_per_prio = sched->queue[queue_index].queue_per_prio; - ring_t *ring = &sched->prio_q[grp][prio][queue_per_prio].ring; + int grp = sched->queue[queue_index].grp; + int prio = sched->queue[queue_index].prio; + int spread = sched->queue[queue_index].spread; + ring_t *ring = &sched->prio_q[grp][prio][spread].ring; - ring_enq(ring, RING_MASK, queue_index); + ring_enq(ring, sched->ring_mask, queue_index); return 0; } @@ -540,13 +590,14 @@ static void schedule_release_atomic(void) uint32_t qi = sched_local.stash_qi; if (qi != PRIO_QUEUE_EMPTY && sched_local.stash_num == 0) { - int grp = sched->queue[qi].grp; - int prio = sched->queue[qi].prio; - int queue_per_prio = sched->queue[qi].queue_per_prio; - ring_t *ring = &sched->prio_q[grp][prio][queue_per_prio].ring; + int grp = sched->queue[qi].grp; + int prio = sched->queue[qi].prio; + int spread = sched->queue[qi].spread; + ring_t *ring = &sched->prio_q[grp][prio][spread].ring; /* Release current atomic queue */ - ring_enq(ring, RING_MASK, qi); + ring_enq(ring, sched->ring_mask, qi); + sched_local.stash_qi = PRIO_QUEUE_EMPTY; } } @@ -773,8 +824,10 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], int prio, i; int ret; int id; - unsigned int max_deq = MAX_DEQ; uint32_t qi; + unsigned int max_deq = MAX_DEQ; + int num_spread = sched->config.num_spread; + uint32_t ring_mask = sched->ring_mask; /* Schedule events */ for (prio = 0; prio < NUM_PRIO; prio++) { @@ -785,14 +838,14 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], /* Select the first ring based on weights */ id = first; - for (i = 0; i < QUEUES_PER_PRIO;) { + for (i = 0; i < num_spread;) { int num; int ordered; odp_queue_t handle; ring_t *ring; int pktin; - if (id >= QUEUES_PER_PRIO) + if (id >= num_spread) id = 0; /* No queues created for this priority queue */ @@ -805,7 +858,7 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], /* Get queue index from the priority queue */ ring = &sched->prio_q[grp][prio][id].ring; - qi = ring_deq(ring, RING_MASK); + qi = ring_deq(ring, ring_mask); /* Priority queue empty */ if (qi == RING_EMPTY) { @@ -854,7 +907,7 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], continue; if (num_pkt == 0 || !stash) { - ring_enq(ring, RING_MASK, qi); + ring_enq(ring, ring_mask, qi); break; } @@ -880,14 +933,14 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], sched_local.ordered.src_queue = qi; /* Continue scheduling ordered queues */ - ring_enq(ring, RING_MASK, qi); + ring_enq(ring, ring_mask, qi); } else if (queue_is_atomic(qi)) { /* Hold queue during atomic access */ sched_local.stash_qi = qi; } else { /* Continue scheduling the queue */ - ring_enq(ring, RING_MASK, qi); + ring_enq(ring, ring_mask, qi); } handle = queue_from_index(qi); @@ -916,7 +969,7 @@ static inline int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[], int i, num_grp; int ret; int first, grp_id; - uint16_t round; + uint16_t spread_round, grp_round; uint32_t epoch; if (sched_local.stash_num) { @@ -933,15 +986,17 @@ static inline int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[], if (odp_unlikely(sched_local.pause)) return 0; - /* Each thread prefers a priority queue. Poll weight table avoids + /* Each thread prefers a priority queue. Spread weight table avoids * starvation of other priority queues on low thread counts. */ - round = sched_local.round + 1; + spread_round = sched_local.spread_round; + grp_round = (sched_local.grp_round++) & (GRP_WEIGHT_TBL_SIZE - 1); - if (odp_unlikely(round == WEIGHT_TBL_SIZE)) - round = 0; + if (odp_unlikely(spread_round + 1 >= sched->max_spread)) + sched_local.spread_round = 0; + else + sched_local.spread_round = spread_round + 1; - sched_local.round = round; - first = sched_local.weight_tbl[round]; + first = sched_local.spread_tbl[spread_round]; epoch = odp_atomic_load_acq_u32(&sched->grp_epoch); num_grp = sched_local.num_grp; @@ -951,7 +1006,7 @@ static inline int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[], sched_local.grp_epoch = epoch; } - grp_id = sched_local.grp_weight[round]; + grp_id = sched_local.grp_weight[grp_round]; /* Schedule queues per group and priority */ for (i = 0; i < num_grp; i++) { diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index f5974442d..023ab1edd 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -21,8 +21,8 @@ #include <odp_internal.h> #include <odp_config_internal.h> #include <odp_debug_internal.h> -#include <_ishm_internal.h> -#include <_ishmpool_internal.h> +#include <odp_ishm_internal.h> +#include <odp_ishmpool_internal.h> #include <odp_align_internal.h> #include <odp_buffer_inlines.h> diff --git a/platform/linux-generic/odp_shared_memory.c b/platform/linux-generic/odp_shared_memory.c index c9b04dfdb..edf261af7 100644 --- a/platform/linux-generic/odp_shared_memory.c +++ b/platform/linux-generic/odp_shared_memory.c @@ -11,7 +11,7 @@ #include <odp/api/std_types.h> #include <odp/api/shared_memory.h> #include <odp/api/plat/strong_types.h> -#include <_ishm_internal.h> +#include <odp_ishm_internal.h> #include <odp_internal.h> #include <string.h> diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 7b9fed72d..0550416b0 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -27,6 +27,7 @@ #include <odp_libconfig_internal.h> #include <protocols/eth.h> +#include <protocols/udp.h> #include <rte_config.h> #include <rte_malloc.h> @@ -48,6 +49,13 @@ #include <rte_string_fns.h> #include <rte_version.h> +/* NUMA is not supported on all platforms */ +#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES +#include <numa.h> +#else +#define numa_num_configured_nodes() 1 +#endif + #if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 0) #define rte_log_set_global_level rte_set_log_level #endif @@ -94,22 +102,16 @@ void refer_constructors(void) } #endif -static int lookup_opt(const char *path, const char *drv_name, int *val) +static int lookup_opt(const char *opt_name, const char *drv_name, int *val) { const char *base = "pktio_dpdk"; - char opt_path[256]; - int ret = 0; - - /* Default option */ - snprintf(opt_path, sizeof(opt_path), "%s.%s", base, path); - ret += _odp_libconfig_lookup_int(opt_path, val); - - /* Driver specific option overrides default option */ - snprintf(opt_path, sizeof(opt_path), "%s.%s.%s", base, drv_name, path); - ret += _odp_libconfig_lookup_int(opt_path, val); + int ret; + ret = _odp_libconfig_lookup_ext_int(base, drv_name, opt_name, val); if (ret == 0) - ODP_ERR("Unable to find DPDK configuration option: %s\n", path); + ODP_ERR("Unable to find DPDK configuration option: %s\n", + opt_name); + return ret; } @@ -419,6 +421,7 @@ MEMPOOL_REGISTER_OPS(ops_stack); #define IP4_CSUM_RESULT(m) (m->ol_flags & PKT_RX_IP_CKSUM_MASK) #define L4_CSUM_RESULT(m) (m->ol_flags & PKT_RX_L4_CKSUM_MASK) #define HAS_L4_PROTO(m, proto) ((m->packet_type & RTE_PTYPE_L4_MASK) == proto) +#define UDP4_CSUM(_p) (((_odp_udphdr_t *)_odp_packet_l4_ptr(_p, NULL))->chksum) #define PKTIN_CSUM_BITS 0x1C @@ -451,6 +454,12 @@ static inline int pkt_set_ol_rx(odp_pktin_config_opt_t *pktin_cfg, if (packet_csum_result == PKT_RX_L4_CKSUM_GOOD) { pkt_hdr->p.input_flags.l4_chksum_done = 1; } else if (packet_csum_result != PKT_RX_L4_CKSUM_UNKNOWN) { + if (pkt_hdr->p.input_flags.ipv4 && + pkt_hdr->p.input_flags.udp && + !UDP4_CSUM(packet_handle(pkt_hdr))) { + pkt_hdr->p.input_flags.l4_chksum_done = 1; + return 0; + } if (pktin_cfg->bit.drop_udp_err) return -1; @@ -648,11 +657,7 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg, if (!ipv4_chksum_pkt && !udp_chksum_pkt && !tcp_chksum_pkt) return; - if (pkt_p->l4_offset == ODP_PACKET_OFFSET_INVALID) - return; - mbuf->l2_len = pkt_p->l3_offset - pkt_p->l2_offset; - mbuf->l3_len = pkt_p->l4_offset - pkt_p->l3_offset; if (l3_proto_v4) mbuf->ol_flags = PKT_TX_IPV4; @@ -663,8 +668,14 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg, mbuf->ol_flags |= PKT_TX_IP_CKSUM; ((struct ipv4_hdr *)l3_hdr)->hdr_checksum = 0; + mbuf->l3_len = _ODP_IPV4HDR_IHL(*(uint8_t *)l3_hdr) * 4; } + if (pkt_p->l4_offset == ODP_PACKET_OFFSET_INVALID) + return; + + mbuf->l3_len = pkt_p->l4_offset - pkt_p->l3_offset; + l4_hdr = (void *)(mbuf_data + pkt_p->l4_offset); if (udp_chksum_pkt) { @@ -1082,6 +1093,7 @@ static int dpdk_pktio_init(void) int32_t masklen; int mem_str_len; int cmd_len; + int numa_nodes; cpu_set_t original_cpuset; struct rte_config *cfg; @@ -1116,21 +1128,29 @@ static int dpdk_pktio_init(void) return -1; } - mem_str_len = snprintf(NULL, 0, "%d", DPDK_MEMORY_MB); + mem_str_len = snprintf(NULL, 0, "%d,", DPDK_MEMORY_MB); + numa_nodes = numa_num_configured_nodes(); + + char mem_str[mem_str_len * numa_nodes]; + + for (i = 0; i < numa_nodes; i++) + sprintf(&mem_str[i * mem_str_len], "%d,", DPDK_MEMORY_MB); + mem_str[mem_str_len * numa_nodes - 1] = '\0'; cmdline = getenv("ODP_PKTIO_DPDK_PARAMS"); if (cmdline == NULL) cmdline = ""; /* masklen includes the terminating null as well */ - cmd_len = strlen("odpdpdk -c -m ") + masklen + mem_str_len + - strlen(cmdline) + strlen(" "); + cmd_len = strlen("odpdpdk -c --socket-mem ") + masklen + + strlen(mem_str) + strlen(cmdline) + strlen(" "); char full_cmd[cmd_len]; /* first argument is facility log, simply bind it to odpdpdk for now.*/ - cmd_len = snprintf(full_cmd, cmd_len, "odpdpdk -c %s -m %d %s", - mask_str, DPDK_MEMORY_MB, cmdline); + cmd_len = snprintf(full_cmd, cmd_len, + "odpdpdk -c %s --socket-mem %s %s", mask_str, + mem_str, cmdline); for (i = 0, dpdk_argc = 1; i < cmd_len; ++i) { if (isspace(full_cmd[i])) diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c index 6dcc7a593..08e3e4bc9 100644 --- a/platform/linux-generic/pktio/ipc.c +++ b/platform/linux-generic/pktio/ipc.c @@ -11,7 +11,7 @@ #include <odp_packet_io_internal.h> #include <odp/api/system_info.h> #include <odp_shm_internal.h> -#include <_ishm_internal.h> +#include <odp_ishm_internal.h> #include <sys/mman.h> #include <sys/stat.h> diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index 7fc0fd61c..fc80e8019 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -15,6 +15,7 @@ #include <odp_debug_internal.h> #include <odp/api/plat/packet_flag_inlines.h> #include <odp/api/hints.h> +#include <odp/api/plat/byteorder_inlines.h> #include <odp_queue_if.h> #include <protocols/eth.h> @@ -32,6 +33,7 @@ static const char pktio_loop_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x01}; static int loopback_stats_reset(pktio_entry_t *pktio_entry); +static int loopback_init_capability(pktio_entry_t *pktio_entry); static int loopback_open(odp_pktio_t id, pktio_entry_t *pktio_entry, const char *devname, odp_pool_t pool ODP_UNUSED) @@ -61,6 +63,7 @@ static int loopback_open(odp_pktio_t id, pktio_entry_t *pktio_entry, return -1; loopback_stats_reset(pktio_entry); + loopback_init_capability(pktio_entry); return 0; } @@ -172,6 +175,85 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, return num_rx; } +#define OL_TX_CHKSUM_PKT(_cfg, _capa, _proto, _ovr_set, _ovr) \ + (_capa && _proto && (_ovr_set ? _ovr : _cfg)) + +static inline int check_proto(void *l3_hdr, + uint32_t l3_len, + odp_bool_t *l3_proto_v4, + uint8_t *l4_proto) +{ + uint8_t l3_proto_ver = _ODP_IPV4HDR_VER(*(uint8_t *)l3_hdr); + + if (l3_proto_ver == _ODP_IPV4 && l3_len >= _ODP_IPV4HDR_LEN) { + _odp_ipv4hdr_t *ip = l3_hdr; + uint16_t frag_offset = _odp_be_to_cpu_16(ip->frag_offset); + + *l3_proto_v4 = 1; + if (!_ODP_IPV4HDR_IS_FRAGMENT(frag_offset)) + *l4_proto = ip->proto; + else + *l4_proto = 255; + + return 0; + } else if (l3_proto_ver == _ODP_IPV6 && l3_len >= _ODP_IPV6HDR_LEN) { + _odp_ipv6hdr_t *ipv6 = l3_hdr; + + *l3_proto_v4 = 0; + *l4_proto = ipv6->next_hdr; + + /* FIXME: check that packet is not a fragment !!! + * Might require parsing headers spanning several segments, so + * not implemented yet. */ + return 0; + } + + return -1; +} + +static inline void loopback_fix_checksums(odp_packet_t pkt, + odp_pktout_config_opt_t *pktout_cfg, + odp_pktout_config_opt_t *pktout_capa) +{ + odp_bool_t l3_proto_v4 = false; + uint8_t l4_proto; + void *l3_hdr; + uint32_t l3_len; + odp_bool_t ipv4_chksum_pkt, udp_chksum_pkt, tcp_chksum_pkt; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + l3_hdr = odp_packet_l3_ptr(pkt, &l3_len); + + if (l3_hdr == NULL || + check_proto(l3_hdr, l3_len, &l3_proto_v4, &l4_proto)) + return; + + ipv4_chksum_pkt = OL_TX_CHKSUM_PKT(pktout_cfg->bit.ipv4_chksum, + pktout_capa->bit.ipv4_chksum, + l3_proto_v4, + pkt_hdr->p.flags.l3_chksum_set, + pkt_hdr->p.flags.l3_chksum); + udp_chksum_pkt = OL_TX_CHKSUM_PKT(pktout_cfg->bit.udp_chksum, + pktout_capa->bit.udp_chksum, + l4_proto == _ODP_IPPROTO_UDP, + pkt_hdr->p.flags.l4_chksum_set, + pkt_hdr->p.flags.l4_chksum); + tcp_chksum_pkt = OL_TX_CHKSUM_PKT(pktout_cfg->bit.tcp_chksum, + pktout_capa->bit.tcp_chksum, + l4_proto == _ODP_IPPROTO_TCP, + pkt_hdr->p.flags.l4_chksum_set, + pkt_hdr->p.flags.l4_chksum); + + if (ipv4_chksum_pkt) + _odp_packet_ipv4_chksum_insert(pkt); + + if (tcp_chksum_pkt) + _odp_packet_tcp_chksum_insert(pkt); + + if (udp_chksum_pkt) + _odp_packet_udp_chksum_insert(pkt); +} + static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, const odp_packet_t pkt_tbl[], int num) { @@ -182,6 +264,9 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, int nb_tx = 0; uint32_t bytes = 0; uint32_t out_octets_tbl[num]; + odp_pktout_config_opt_t *pktout_cfg = &pktio_entry->s.config.pktout; + odp_pktout_config_opt_t *pktout_capa = + &pktio_entry->s.capa.config.pktout; if (odp_unlikely(num > QUEUE_MULTI_MAX)) num = QUEUE_MULTI_MAX; @@ -218,6 +303,9 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, packet_subtype_set(pkt_tbl[i], ODP_EVENT_PACKET_BASIC); } + for (i = 0; i < nb_tx; ++i) + loopback_fix_checksums(pkt_tbl[i], pktout_cfg, pktout_capa); + odp_ticketlock_lock(&pktio_entry->s.txl); queue = queue_fn->from_ext(pktio_entry->s.pkt_loop.loopq); @@ -255,9 +343,10 @@ static int loopback_link_status(pktio_entry_t *pktio_entry ODP_UNUSED) return 1; } -static int loopback_capability(pktio_entry_t *pktio_entry ODP_UNUSED, - odp_pktio_capability_t *capa) +static int loopback_init_capability(pktio_entry_t *pktio_entry) { + odp_pktio_capability_t *capa = &pktio_entry->s.capa; + memset(capa, 0, sizeof(odp_pktio_capability_t)); capa->max_input_queues = 1; @@ -267,9 +356,26 @@ static int loopback_capability(pktio_entry_t *pktio_entry ODP_UNUSED, odp_pktio_config_init(&capa->config); capa->config.pktin.bit.ts_all = 1; capa->config.pktin.bit.ts_ptp = 1; + capa->config.pktout.bit.ipv4_chksum = 1; + capa->config.pktout.bit.tcp_chksum = 1; + capa->config.pktout.bit.udp_chksum = 1; capa->config.inbound_ipsec = 1; capa->config.outbound_ipsec = 1; + capa->config.pktout.bit.ipv4_chksum_ena = + capa->config.pktout.bit.ipv4_chksum; + capa->config.pktout.bit.udp_chksum_ena = + capa->config.pktout.bit.udp_chksum; + capa->config.pktout.bit.tcp_chksum_ena = + capa->config.pktout.bit.tcp_chksum; + + return 0; +} + +static int loopback_capability(pktio_entry_t *pktio_entry ODP_UNUSED, + odp_pktio_capability_t *capa) +{ + *capa = pktio_entry->s.capa; return 0; } diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index 4276d4bc1..835d81ebb 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -27,6 +27,8 @@ #include <odp_classification_datamodel.h> #include <odp_classification_inlines.h> #include <odp_classification_internal.h> +#include <odp_libconfig_internal.h> + #include <inttypes.h> @@ -46,6 +48,49 @@ static int disable_pktio; /** !0 this pktio disabled, 0 enabled */ static int netmap_stats_reset(pktio_entry_t *pktio_entry); +static int lookup_opt(const char *opt_name, const char *drv_name, int *val) +{ + const char *base = "pktio_netmap"; + int ret; + + ret = _odp_libconfig_lookup_ext_int(base, drv_name, opt_name, val); + if (ret == 0) + ODP_ERR("Unable to find netmap configuration option: %s\n", + opt_name); + + return ret; +} + +static int init_options(pktio_entry_t *pktio_entry) +{ + netmap_opt_t *opt = &pktio_entry->s.pkt_nm.opt; + + if (!lookup_opt("nr_rx_slots", "virt", + &opt->nr_rx_slots)) + return -1; + if (opt->nr_rx_slots < 0 || + opt->nr_rx_slots > 4096) { + ODP_ERR("Invalid number of RX slots\n"); + return -1; + } + + if (!lookup_opt("nr_tx_slots", "virt", + &opt->nr_tx_slots)) + return -1; + if (opt->nr_tx_slots < 0 || + opt->nr_tx_slots > 4096) { + ODP_ERR("Invalid number of TX slots\n"); + return -1; + } + + ODP_PRINT("netmap interface: %s\n", + pktio_entry->s.pkt_nm.if_name); + ODP_PRINT(" num_rx_desc: %d\n", opt->nr_rx_slots); + ODP_PRINT(" num_tx_desc: %d\n", opt->nr_tx_slots); + + return 0; +} + static int netmap_do_ioctl(pktio_entry_t *pktio_entry, unsigned long cmd, int subcmd) { @@ -363,6 +408,12 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, netdev); snprintf(pkt_nm->if_name, sizeof(pkt_nm->if_name), "%s", netdev); + /* Initialize runtime options */ + if (init_options(pktio_entry)) { + ODP_ERR("Initializing runtime options failed\n"); + return -1; + } + /* Dummy open here to check if netmap module is available and to read * capability info. */ desc = nm_open(pkt_nm->nm_name, NULL, 0, NULL); @@ -526,6 +577,12 @@ static int netmap_start(pktio_entry_t *pktio_entry) base_desc.self = &base_desc; base_desc.mem = NULL; + if (pktio_entry->s.pkt_nm.is_virtual) { + base_desc.req.nr_rx_slots = + pktio_entry->s.pkt_nm.opt.nr_rx_slots; + base_desc.req.nr_tx_slots = + pktio_entry->s.pkt_nm.opt.nr_tx_slots; + } base_desc.req.nr_ringid = 0; if ((base_desc.req.nr_flags & NR_REG_MASK) == NR_REG_ALL_NIC || (base_desc.req.nr_flags & NR_REG_MASK) == NR_REG_ONE_NIC) { @@ -539,6 +596,8 @@ static int netmap_start(pktio_entry_t *pktio_entry) /* Only the first rx descriptor does mmap */ desc_ring = pkt_nm->rx_desc_ring; flags = NM_OPEN_IFNAME | NETMAP_NO_TX_POLL; + if (pktio_entry->s.pkt_nm.is_virtual) + flags |= NM_OPEN_RING_CFG; desc_ring[0].s.desc[0] = nm_open(pkt_nm->nm_name, NULL, flags, &base_desc); if (desc_ring[0].s.desc[0] == NULL) { @@ -547,6 +606,8 @@ static int netmap_start(pktio_entry_t *pktio_entry) } /* Open rest of the rx descriptors (one per netmap ring) */ flags = NM_OPEN_IFNAME | NETMAP_NO_TX_POLL | NM_OPEN_NO_MMAP; + if (pktio_entry->s.pkt_nm.is_virtual) + flags |= NM_OPEN_RING_CFG; for (i = 0; i < pktio_entry->s.num_in_queue; i++) { for (j = desc_ring[i].s.first; j <= desc_ring[i].s.last; j++) { if (i == 0 && j == 0) { /* First already opened */ @@ -568,6 +629,8 @@ static int netmap_start(pktio_entry_t *pktio_entry) /* Open tx descriptors */ desc_ring = pkt_nm->tx_desc_ring; flags = NM_OPEN_IFNAME | NM_OPEN_NO_MMAP; + if (pktio_entry->s.pkt_nm.is_virtual) + flags |= NM_OPEN_RING_CFG; if ((base_desc.req.nr_flags & NR_REG_MASK) == NR_REG_ALL_NIC) { base_desc.req.nr_flags &= ~NR_REG_ALL_NIC; |