diff options
Diffstat (limited to 'platform/linux-generic/pktio')
| -rw-r--r-- | platform/linux-generic/pktio/dpdk.c | 254 | ||||
| -rw-r--r-- | platform/linux-generic/pktio/dpdk_parse.c | 28 | ||||
| -rw-r--r-- | platform/linux-generic/pktio/loop.c | 96 | ||||
| -rw-r--r-- | platform/linux-generic/pktio/netmap.c | 53 | ||||
| -rw-r--r-- | platform/linux-generic/pktio/null.c | 4 | ||||
| -rw-r--r-- | platform/linux-generic/pktio/pcap.c | 69 | ||||
| -rw-r--r-- | platform/linux-generic/pktio/pktio_common.c | 12 | ||||
| -rw-r--r-- | platform/linux-generic/pktio/socket.c | 67 | ||||
| -rw-r--r-- | platform/linux-generic/pktio/socket_mmap.c | 44 | ||||
| -rw-r--r-- | platform/linux-generic/pktio/tap.c | 42 |
10 files changed, 358 insertions, 311 deletions
diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 815ec61b9..030560b0d 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -57,12 +57,14 @@ #include <rte_string_fns.h> #include <rte_version.h> -#if RTE_VERSION < RTE_VERSION_NUM(19, 8, 0, 0) - #define rte_ether_addr ether_addr - #define rte_ipv4_hdr ipv4_hdr - #define rte_ipv6_hdr ipv6_hdr - #define rte_tcp_hdr tcp_hdr - #define rte_udp_hdr udp_hdr +#if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0) + #define RTE_MBUF_F_RX_RSS_HASH PKT_RX_RSS_HASH + #define RTE_MBUF_F_TX_IPV4 PKT_TX_IPV4 + #define RTE_MBUF_F_TX_IPV6 PKT_TX_IPV6 + #define RTE_MBUF_F_TX_IP_CKSUM PKT_TX_IP_CKSUM + #define RTE_MBUF_F_TX_UDP_CKSUM PKT_TX_UDP_CKSUM + #define RTE_MBUF_F_TX_TCP_CKSUM PKT_TX_TCP_CKSUM + #define RTE_MEMPOOL_REGISTER_OPS MEMPOOL_REGISTER_OPS #endif /* NUMA is not supported on all platforms */ @@ -72,17 +74,7 @@ #define numa_num_configured_nodes() 1 #endif -#if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 0) -#define rte_log_set_global_level rte_set_log_level -#endif - -/* Release notes v19.11: "Changed the mempool allocation behaviour - * so that objects no longer cross pages by default" */ -#if RTE_VERSION >= RTE_VERSION_NUM(19, 11, 0, 0) -#define MEMPOOL_FLAGS MEMPOOL_F_NO_IOVA_CONTIG -#else #define MEMPOOL_FLAGS 0 -#endif #if _ODP_DPDK_ZERO_COPY ODP_STATIC_ASSERT(CONFIG_PACKET_HEADROOM == RTE_PKTMBUF_HEADROOM, @@ -107,11 +99,7 @@ ODP_STATIC_ASSERT((DPDK_NB_MBUF % DPDK_MEMPOOL_CACHE_SIZE == 0) && #define DPDK_MIN_RX_BURST 4 /* Limits for setting link MTU */ -#if RTE_VERSION >= RTE_VERSION_NUM(19, 11, 0, 0) #define DPDK_MTU_MIN (RTE_ETHER_MIN_MTU + _ODP_ETHHDR_LEN) -#else -#define DPDK_MTU_MIN (68 + _ODP_ETHHDR_LEN) -#endif #define DPDK_MTU_MAX (9000 + _ODP_ETHHDR_LEN) /** DPDK runtime configuration options */ @@ -334,12 +322,16 @@ static void pktmbuf_init(struct rte_mempool *mp, void *opaque_arg ODP_UNUSED, static struct rte_mempool *mbuf_pool_create(const char *name, pool_t *pool_entry) { + odp_shm_info_t shm_info; struct rte_mempool *mp = NULL; struct rte_pktmbuf_pool_private mbp_priv; struct rte_mempool_objsz sz; unsigned int elt_size = pool_entry->dpdk_elt_size; - unsigned int num = pool_entry->num; + unsigned int num = pool_entry->num, populated = 0; uint32_t total_size; + uint64_t page_size, offset = 0, remainder = 0; + uint8_t *addr; + int ret; if (!(pool_entry->mem_from_huge_pages)) { ODP_ERR("DPDK requires memory is allocated from huge pages\n"); @@ -348,11 +340,17 @@ static struct rte_mempool *mbuf_pool_create(const char *name, if (pool_entry->seg_len < RTE_MBUF_DEFAULT_BUF_SIZE) { ODP_ERR("Some NICs need at least %dB buffers to not segment " - "standard ethernet frames. Increase pool seg_len.\n", - RTE_MBUF_DEFAULT_BUF_SIZE); + "standard ethernet frames. Increase pool seg_len.\n", + RTE_MBUF_DEFAULT_BUF_SIZE); goto fail; } + if (odp_shm_info(pool_entry->shm, &shm_info)) { + ODP_ERR("Failed to query SHM info.\n"); + goto fail; + } + + page_size = shm_info.page_size; total_size = rte_mempool_calc_obj_size(elt_size, MEMPOOL_FLAGS, &sz); if (total_size != pool_entry->block_size) { ODP_ERR("DPDK pool block size not matching to ODP pool: " @@ -361,10 +359,7 @@ static struct rte_mempool *mbuf_pool_create(const char *name, goto fail; } - /* Skipped buffers have to be taken into account to populate pool - * properly. */ - mp = rte_mempool_create_empty(name, num + pool_entry->skipped_blocks, - elt_size, cache_size(num), + mp = rte_mempool_create_empty(name, num, elt_size, cache_size(num), sizeof(struct rte_pktmbuf_pool_private), rte_socket_id(), MEMPOOL_FLAGS); if (mp == NULL) { @@ -380,16 +375,34 @@ static struct rte_mempool *mbuf_pool_create(const char *name, } mbp_priv.mbuf_data_room_size = pool_entry->headroom + - pool_entry->seg_len; + pool_entry->seg_len + pool_entry->tailroom; mbp_priv.mbuf_priv_size = RTE_ALIGN(sizeof(odp_packet_hdr_t), RTE_MBUF_PRIV_ALIGN); rte_pktmbuf_pool_init(mp, &mbp_priv); - num = rte_mempool_populate_iova(mp, (char *)pool_entry->base_addr, - RTE_BAD_IOVA, pool_entry->shm_size, - NULL, NULL); - if (num <= 0) { - ODP_ERR("Failed to populate mempool: %d\n", num); + /* DPDK expects buffers that would be crossing a hugepage boundary to be aligned to the + * boundary. This isn't the case with ODP pools as boundary-crossing buffers are skipped + * and unused but still part of the pool. Thus, populate the mempool with several virtually + * and physically contiguous chunks as dictated by the skipped buffers. */ + for (uint64_t i = 0; i < pool_entry->shm_size; i += page_size) { + remainder = (page_size - offset) % total_size; + addr = pool_entry->base_addr + i + offset; + ret = rte_mempool_populate_iova(mp, (char *)addr, rte_mem_virt2iova(addr), + page_size - remainder - offset, + NULL, NULL); + + if (ret <= 0) { + ODP_ERR("Failed to populate mempool: %d\n", ret); + goto fail; + } + + populated += ret; + offset = remainder ? total_size - remainder : 0; + } + + if (populated != num) { + ODP_ERR("Failed to populate mempool with all requested blocks, populated: %u, " + "requested: %u\n", populated, num); goto fail; } @@ -556,7 +569,7 @@ static struct rte_mempool_ops odp_pool_ops = { .get_count = pool_get_count }; -MEMPOOL_REGISTER_OPS(odp_pool_ops) +RTE_MEMPOOL_REGISTER_OPS(odp_pool_ops) static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], @@ -574,9 +587,9 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, pkt_dpdk_t *pkt_dpdk = pkt_priv(pktio_entry); odp_pool_t pool = pkt_dpdk->pool; odp_pktin_config_opt_t pktin_cfg = pktio_entry->s.config.pktin; - odp_proto_layer_t parse_layer = pktio_entry->s.config.parser.layer; odp_pktio_t input = pktio_entry->s.handle; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; /* Allocate maximum sized packets */ max_len = pkt_dpdk->data_room; @@ -591,8 +604,6 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, } for (i = 0; i < num; i++) { - odp_packet_hdr_t parsed_hdr; - mbuf = mbuf_table[i]; if (odp_unlikely(mbuf->nb_segs != 1)) { ODP_ERR("Segmented buffers not supported\n"); @@ -603,31 +614,33 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, odp_prefetch(data); pkt_len = rte_pktmbuf_pkt_len(mbuf); + pkt = pkt_table[i]; + pkt_hdr = packet_hdr(pkt); - if (pktio_cls_enabled(pktio_entry)) { + if (layer) { uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; - packet_parse_reset(&parsed_hdr, 1); - packet_set_len(&parsed_hdr, pkt_len); - if (_odp_dpdk_packet_parse_common(&parsed_hdr.p, data, - pkt_len, pkt_len, - mbuf, - ODP_PROTO_LAYER_ALL, - supported_ptypes, + packet_parse_reset(pkt_hdr, 1); + if (_odp_dpdk_packet_parse_common(&pkt_hdr->p, data, + pkt_len, pkt_len, mbuf, + layer, supported_ptypes, pktin_cfg)) { odp_packet_free(pkt_table[i]); rte_pktmbuf_free(mbuf); continue; } - if (_odp_cls_classify_packet(pktio_entry, - (const uint8_t *)data, - pkt_len, pkt_len, &pool, - &parsed_hdr, false)) - goto fail; + + if (pktio_cls_enabled(pktio_entry)) { + if (_odp_cls_classify_packet(pktio_entry, + (const uint8_t *)data, + &pool, pkt_hdr)) { + odp_packet_free(pkt_table[i]); + rte_pktmbuf_free(mbuf); + continue; + } + } } - pkt = pkt_table[i]; - pkt_hdr = packet_hdr(pkt); pull_tail(pkt_hdr, max_len - pkt_len); if (frame_offset) pull_head(pkt_hdr, frame_offset); @@ -637,22 +650,7 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, pkt_hdr->input = input; - if (pktio_cls_enabled(pktio_entry)) { - _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr); - } else if (parse_layer != ODP_PROTO_LAYER_NONE) { - uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; - - if (_odp_dpdk_packet_parse_layer(pkt_hdr, mbuf, - parse_layer, - supported_ptypes, - pktin_cfg)) { - odp_packet_free(pkt); - rte_pktmbuf_free(mbuf); - continue; - } - } - - if (mbuf->ol_flags & PKT_RX_RSS_HASH) + if (mbuf->ol_flags & RTE_MBUF_F_RX_RSS_HASH) packet_set_flow_hash(pkt_hdr, mbuf->hash.rss); packet_set_ts(pkt_hdr, ts); @@ -753,12 +751,12 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg, mbuf->l2_len = pkt_p->l3_offset - pkt_p->l2_offset; if (l3_proto_v4) - mbuf->ol_flags = PKT_TX_IPV4; + mbuf->ol_flags = RTE_MBUF_F_TX_IPV4; else - mbuf->ol_flags = PKT_TX_IPV6; + mbuf->ol_flags = RTE_MBUF_F_TX_IPV6; if (ipv4_chksum_pkt) { - mbuf->ol_flags |= PKT_TX_IP_CKSUM; + mbuf->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM; ((struct rte_ipv4_hdr *)l3_hdr)->hdr_checksum = 0; mbuf->l3_len = _ODP_IPV4HDR_IHL(*(uint8_t *)l3_hdr) * 4; @@ -772,12 +770,12 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg, l4_hdr = (void *)(mbuf_data + pkt_p->l4_offset); if (udp_chksum_pkt) { - mbuf->ol_flags |= PKT_TX_UDP_CKSUM; + mbuf->ol_flags |= RTE_MBUF_F_TX_UDP_CKSUM; ((struct rte_udp_hdr *)l4_hdr)->dgram_cksum = phdr_csum(l3_proto_v4, l3_hdr, mbuf->ol_flags); } else if (tcp_chksum_pkt) { - mbuf->ol_flags |= PKT_TX_TCP_CKSUM; + mbuf->ol_flags |= RTE_MBUF_F_TX_TCP_CKSUM; ((struct rte_tcp_hdr *)l4_hdr)->cksum = phdr_csum(l3_proto_v4, l3_hdr, mbuf->ol_flags); @@ -863,9 +861,9 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, int i, nb_pkts; odp_pool_t pool; odp_pktin_config_opt_t pktin_cfg; - odp_proto_layer_t parse_layer; odp_pktio_t input; pkt_dpdk_t *pkt_dpdk; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; prefetch_pkt(mbuf_table[0]); @@ -874,15 +872,12 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, pool = pkt_dpdk->pool; set_flow_hash = pkt_dpdk->opt.set_flow_hash; pktin_cfg = pktio_entry->s.config.pktin; - parse_layer = pktio_entry->s.config.parser.layer; input = pktio_entry->s.handle; if (odp_likely(mbuf_num > 1)) prefetch_pkt(mbuf_table[1]); for (i = 0; i < mbuf_num; i++) { - odp_packet_hdr_t parsed_hdr; - if (odp_likely((i + 2) < mbuf_num)) prefetch_pkt(mbuf_table[i + 2]); @@ -895,30 +890,27 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, data = rte_pktmbuf_mtod(mbuf, char *); pkt_len = rte_pktmbuf_pkt_len(mbuf); - pkt_hdr = pkt_hdr_from_mbuf(mbuf); + packet_init(pkt_hdr, pkt_len); - if (pktio_cls_enabled(pktio_entry)) { + if (layer) { uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; - packet_parse_reset(&parsed_hdr, 1); - packet_set_len(&parsed_hdr, pkt_len); - if (_odp_dpdk_packet_parse_common(&parsed_hdr.p, data, - pkt_len, pkt_len, - mbuf, - ODP_PROTO_LAYER_ALL, - supported_ptypes, + if (_odp_dpdk_packet_parse_common(&pkt_hdr->p, data, + pkt_len, pkt_len, mbuf, + layer, supported_ptypes, pktin_cfg)) { rte_pktmbuf_free(mbuf); continue; } - if (_odp_cls_classify_packet(pktio_entry, - (const uint8_t *)data, - pkt_len, pkt_len, &pool, - &parsed_hdr, false)) { - ODP_ERR("Unable to classify packet\n"); - rte_pktmbuf_free(mbuf); - continue; + + if (pktio_cls_enabled(pktio_entry)) { + if (_odp_cls_classify_packet(pktio_entry, + (const uint8_t *)data, + &pool, pkt_hdr)) { + rte_pktmbuf_free(mbuf); + continue; + } } } @@ -926,23 +918,9 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, * are supported. */ pkt_hdr->seg_data = data; - packet_init(pkt_hdr, pkt_len); pkt_hdr->input = input; - if (pktio_cls_enabled(pktio_entry)) { - _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr); - } else if (parse_layer != ODP_PROTO_LAYER_NONE) { - uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; - - if (_odp_dpdk_packet_parse_layer(pkt_hdr, mbuf, - parse_layer, - supported_ptypes, - pktin_cfg)) { - rte_pktmbuf_free(mbuf); - continue; - } - } - if (set_flow_hash && (mbuf->ol_flags & PKT_RX_RSS_HASH)) + if (set_flow_hash && (mbuf->ol_flags & RTE_MBUF_F_RX_RSS_HASH)) packet_set_flow_hash(pkt_hdr, mbuf->hash.rss); packet_set_ts(pkt_hdr, ts); @@ -1227,11 +1205,6 @@ static int dpdk_close(pktio_entry_t *pktio_entry) rte_pktmbuf_free(pkt_dpdk->rx_cache[i].s.pkt[idx++]); } -#if RTE_VERSION < RTE_VERSION_NUM(17, 8, 0, 0) - if (pktio_entry->s.state != PKTIO_STATE_OPENED) - rte_eth_dev_close(pkt_dpdk->port_id); -#endif - return 0; } @@ -1381,18 +1354,6 @@ static void dpdk_mempool_free(struct rte_mempool *mp, void *arg ODP_UNUSED) rte_mempool_free(mp); } -/* RTE_ETH_FOREACH_DEV was introduced in v17.8, but causes a build error in - * v18.2 (only a warning, but our build system treats warnings as errors). */ -#if (RTE_VERSION >= RTE_VERSION_NUM(18, 2, 0, 0)) && \ - (RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0)) - #define ETH_FOREACH_DEV(p) \ - for (p = rte_eth_find_next(0); \ - (unsigned int)p < (unsigned int)RTE_MAX_ETHPORTS; \ - p = rte_eth_find_next(p + 1)) -#elif RTE_VERSION >= RTE_VERSION_NUM(17, 8, 0, 0) - #define ETH_FOREACH_DEV(p) RTE_ETH_FOREACH_DEV(p) -#endif - static int dpdk_pktio_term(void) { uint16_t port_id; @@ -1400,11 +1361,9 @@ static int dpdk_pktio_term(void) if (!odp_global_rw->dpdk_initialized) return 0; -#if RTE_VERSION >= RTE_VERSION_NUM(17, 8, 0, 0) - ETH_FOREACH_DEV(port_id) { + RTE_ETH_FOREACH_DEV(port_id) { rte_eth_dev_close(port_id); } -#endif if (!_ODP_DPDK_ZERO_COPY) rte_mempool_walk(dpdk_mempool_free, NULL); @@ -1574,16 +1533,18 @@ static int dpdk_init_capability(pktio_entry_t *pktio_entry, /* Check if setting MTU is supported */ ret = rte_eth_dev_set_mtu(pkt_dpdk->port_id, pkt_dpdk->mtu - _ODP_ETHHDR_LEN); - if (ret == 0) { + /* From DPDK 21.11 onwards, calling rte_eth_dev_set_mtu() before device is configured with + * rte_eth_dev_configure() will result in failure. The least hacky (unfortunately still + * very hacky) way to continue checking the support is to take into account that the + * function will fail earlier with -ENOTSUP if MTU setting is not supported by device than + * if the device was not yet configured. */ + if (ret != -ENOTSUP) { capa->set_op.op.maxlen = 1; capa->maxlen.equal = true; capa->maxlen.min_input = DPDK_MTU_MIN; capa->maxlen.max_input = pkt_dpdk->mtu_max; capa->maxlen.min_output = DPDK_MTU_MIN; capa->maxlen.max_output = pkt_dpdk->mtu_max; - } else if (ret != -ENOTSUP) { - ODP_ERR("Failed to set interface MTU: %d\n", ret); - return -1; } ptype_cnt = rte_eth_dev_get_supported_ptypes(pkt_dpdk->port_id, @@ -1669,15 +1630,6 @@ static int dpdk_init_capability(pktio_entry_t *pktio_entry, * mode change. Use system call for them. */ static void promisc_mode_check(pkt_dpdk_t *pkt_dpdk) { -#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0) - /* Enable and disable calls do not have return value */ - rte_eth_promiscuous_enable(pkt_dpdk->port_id); - - if (!rte_eth_promiscuous_get(pkt_dpdk->port_id)) - pkt_dpdk->vdev_sysc_promisc = 1; - - rte_eth_promiscuous_disable(pkt_dpdk->port_id); -#else int ret; ret = rte_eth_promiscuous_enable(pkt_dpdk->port_id); @@ -1689,7 +1641,6 @@ static void promisc_mode_check(pkt_dpdk_t *pkt_dpdk) if (ret) pkt_dpdk->vdev_sysc_promisc = 1; -#endif } static int dpdk_open(odp_pktio_t id ODP_UNUSED, @@ -1735,18 +1686,17 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED, pkt_dpdk->pool = pool; - /* rte_eth_dev_count() was removed in v18.05 */ -#if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0) - if (rte_eth_dev_count() == 0) { -#else if (rte_eth_dev_count_avail() == 0) { -#endif ODP_ERR("No DPDK ports found\n"); return -1; } memset(&dev_info, 0, sizeof(struct rte_eth_dev_info)); - rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info); + ret = rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info); + if (ret) { + ODP_ERR("Failed to read device info: %d\n", ret); + return -1; + } /* Initialize runtime options */ if (init_options(pktio_entry, &dev_info)) { @@ -1765,18 +1715,10 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED, promisc_mode_check(pkt_dpdk); -#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0) - ret = 0; - if (pkt_dpdk->opt.multicast_en) - rte_eth_allmulticast_enable(pkt_dpdk->port_id); - else - rte_eth_allmulticast_disable(pkt_dpdk->port_id); -#else if (pkt_dpdk->opt.multicast_en) ret = rte_eth_allmulticast_enable(pkt_dpdk->port_id); else ret = rte_eth_allmulticast_disable(pkt_dpdk->port_id); -#endif /* Not supported by all PMDs, so ignore the return value */ if (ret) diff --git a/platform/linux-generic/pktio/dpdk_parse.c b/platform/linux-generic/pktio/dpdk_parse.c index 0984f06c5..f28f18921 100644 --- a/platform/linux-generic/pktio/dpdk_parse.c +++ b/platform/linux-generic/pktio/dpdk_parse.c @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -26,9 +26,19 @@ #if defined(__PPC64__) && defined(vector) #undef vector #endif +#include <rte_version.h> + +#if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0) + #define RTE_MBUF_F_RX_IP_CKSUM_MASK PKT_RX_IP_CKSUM_MASK + #define RTE_MBUF_F_RX_IP_CKSUM_GOOD PKT_RX_IP_CKSUM_GOOD + #define RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN PKT_RX_IP_CKSUM_UNKNOWN + #define RTE_MBUF_F_RX_L4_CKSUM_MASK PKT_RX_L4_CKSUM_MASK + #define RTE_MBUF_F_RX_L4_CKSUM_GOOD PKT_RX_L4_CKSUM_GOOD + #define RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN PKT_RX_L4_CKSUM_UNKNOWN +#endif -#define IP4_CSUM_RESULT(ol_flags) (ol_flags & PKT_RX_IP_CKSUM_MASK) -#define L4_CSUM_RESULT(ol_flags) (ol_flags & PKT_RX_L4_CKSUM_MASK) +#define IP4_CSUM_RESULT(ol_flags) ((ol_flags) & RTE_MBUF_F_RX_IP_CKSUM_MASK) +#define L4_CSUM_RESULT(ol_flags) ((ol_flags) & RTE_MBUF_F_RX_L4_CKSUM_MASK) /** Parser helper function for Ethernet packets */ static inline uint16_t dpdk_parse_eth(packet_parser_t *prs, @@ -174,9 +184,9 @@ static inline uint8_t dpdk_parse_ipv4(packet_parser_t *prs, if (do_csum) { uint64_t packet_csum_result = IP4_CSUM_RESULT(mbuf_ol); - if (packet_csum_result == PKT_RX_IP_CKSUM_GOOD) { + if (packet_csum_result == RTE_MBUF_F_RX_IP_CKSUM_GOOD) { prs->input_flags.l3_chksum_done = 1; - } else if (packet_csum_result != PKT_RX_IP_CKSUM_UNKNOWN) { + } else if (packet_csum_result != RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN) { prs->input_flags.l3_chksum_done = 1; prs->flags.ip_err = 1; prs->flags.l3_chksum_err = 1; @@ -302,9 +312,9 @@ static inline void dpdk_parse_tcp(packet_parser_t *prs, if (do_csum) { uint64_t packet_csum_result = L4_CSUM_RESULT(mbuf_ol); - if (packet_csum_result == PKT_RX_L4_CKSUM_GOOD) { + if (packet_csum_result == RTE_MBUF_F_RX_L4_CKSUM_GOOD) { prs->input_flags.l4_chksum_done = 1; - } else if (packet_csum_result != PKT_RX_L4_CKSUM_UNKNOWN) { + } else if (packet_csum_result != RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN) { prs->input_flags.l4_chksum_done = 1; prs->flags.tcp_err = 1; prs->flags.l4_chksum_err = 1; @@ -332,9 +342,9 @@ static inline void dpdk_parse_udp(packet_parser_t *prs, if (do_csum) { uint64_t packet_csum_result = L4_CSUM_RESULT(mbuf_ol); - if (packet_csum_result == PKT_RX_L4_CKSUM_GOOD) { + if (packet_csum_result == RTE_MBUF_F_RX_L4_CKSUM_GOOD) { prs->input_flags.l4_chksum_done = 1; - } else if (packet_csum_result != PKT_RX_L4_CKSUM_UNKNOWN) { + } else if (packet_csum_result != RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN) { if (prs->input_flags.ipv4 && !udp->chksum) { prs->input_flags.l4_chksum_done = 1; } else { diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index b3232cded..3e21efecd 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -18,6 +18,7 @@ #include <odp/api/plat/packet_flag_inlines.h> #include <odp/api/plat/queue_inlines.h> +#include <odp_parse_internal.h> #include <odp_classification_internal.h> #include <odp_debug_internal.h> #include <odp_errno_define.h> @@ -26,6 +27,7 @@ #include <odp_ipsec_internal.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> +#include <odp_macros_internal.h> #include <odp_queue_if.h> #include <protocols/eth.h> @@ -158,7 +160,11 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_time_t ts_val; odp_time_t *ts = NULL; int num_rx = 0; - int failed = 0; + int packets = 0, errors = 0; + uint32_t octets = 0; + const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; if (odp_unlikely(num > QUEUE_MULTI_MAX)) num = QUEUE_MULTI_MAX; @@ -168,8 +174,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, queue = pkt_priv(pktio_entry)->loopq; nbr = odp_queue_deq_multi(queue, (odp_event_t *)hdr_tbl, num); - if (pktio_entry->s.config.pktin.bit.ts_all || - pktio_entry->s.config.pktin.bit.ts_ptp) { + if (opt.bit.ts_all || opt.bit.ts_ptp) { ts_val = odp_time_global(); ts = &ts_val; } @@ -181,61 +186,64 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pkt_len = odp_packet_len(pkt); pkt_hdr = packet_hdr(pkt); - packet_parse_reset(pkt_hdr, 1); - if (pktio_cls_enabled(pktio_entry)) { - odp_packet_t new_pkt; - odp_pool_t new_pool; + if (layer) { uint8_t *pkt_addr; - uint8_t buf[PACKET_PARSE_SEG_LEN]; + uint8_t buf[PARSE_BYTES]; int ret; uint32_t seg_len = odp_packet_seg_len(pkt); + uint64_t l4_part_sum = 0; /* Make sure there is enough data for the packet * parser in the case of a segmented packet. */ - if (odp_unlikely(seg_len < PACKET_PARSE_SEG_LEN && - pkt_len > PACKET_PARSE_SEG_LEN)) { - odp_packet_copy_to_mem(pkt, 0, - PACKET_PARSE_SEG_LEN, - buf); - seg_len = PACKET_PARSE_SEG_LEN; + if (odp_unlikely(seg_len < PARSE_BYTES && + pkt_len > seg_len)) { + seg_len = MIN(pkt_len, PARSE_BYTES); + odp_packet_copy_to_mem(pkt, 0, seg_len, buf); pkt_addr = buf; } else { pkt_addr = odp_packet_data(pkt); } - ret = _odp_cls_classify_packet(pktio_entry, pkt_addr, - pkt_len, seg_len, - &new_pool, pkt_hdr, true); - if (ret) { - failed++; + packet_parse_reset(pkt_hdr, 1); + ret = _odp_packet_parse_common(&pkt_hdr->p, pkt_addr, pkt_len, + seg_len, layer, chksums, + &l4_part_sum, opt); + if (ret) + errors++; + + if (ret < 0) { odp_packet_free(pkt); continue; } - if (new_pool != odp_packet_pool(pkt)) { - new_pkt = odp_packet_copy(pkt, new_pool); - - odp_packet_free(pkt); + if (pktio_cls_enabled(pktio_entry)) { + odp_packet_t new_pkt; + odp_pool_t new_pool; - if (new_pkt == ODP_PACKET_INVALID) { - failed++; + ret = _odp_cls_classify_packet(pktio_entry, pkt_addr, + &new_pool, pkt_hdr); + if (ret) { + odp_packet_free(pkt); continue; } - pkt = new_pkt; - pkt_hdr = packet_hdr(new_pkt); + if (new_pool != odp_packet_pool(pkt)) { + new_pkt = odp_packet_copy(pkt, new_pool); + + odp_packet_free(pkt); + + if (new_pkt == ODP_PACKET_INVALID) { + pktio_entry->s.stats.in_discards++; + continue; + } + + pkt = new_pkt; + pkt_hdr = packet_hdr(new_pkt); + } } - } else { - odp_packet_parse_param_t param; - - /* - * Use odp_packet_parse() which can handle segmented - * packets. - */ - param.proto = ODP_PROTO_ETH; - param.last_layer = pktio_entry->s.config.parser.layer; - param.chksums = pktio_entry->s.in_chksums; - odp_packet_parse(packet_handle(pkt_hdr), 0, ¶m); + + if (layer >= ODP_PROTO_LAYER_L4) + _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); } packet_set_ts(pkt_hdr, ts); @@ -247,12 +255,17 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_packet_has_ipsec(pkt)) _odp_ipsec_try_inline(&pkt); - pktio_entry->s.stats.in_octets += pkt_len; + if (!pkt_hdr->p.flags.all.error) { + octets += pkt_len; + packets++; + } + pkts[num_rx++] = pkt; } - pktio_entry->s.stats.in_errors += failed; - pktio_entry->s.stats.in_packets += num_rx - failed; + pktio_entry->s.stats.in_octets += octets; + pktio_entry->s.stats.in_packets += packets; + pktio_entry->s.stats.in_errors += errors; odp_ticketlock_unlock(&pktio_entry->s.rxl); @@ -533,6 +546,7 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry) capa->stats.pktio.counter.in_octets = 1; capa->stats.pktio.counter.in_packets = 1; capa->stats.pktio.counter.in_errors = 1; + capa->stats.pktio.counter.in_discards = 1; capa->stats.pktio.counter.out_octets = 1; capa->stats.pktio.counter.out_packets = 1; capa->stats.pktin_queue.counter.octets = 1; diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index 0b1345e50..94b88e21e 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -16,6 +16,7 @@ #include <odp/api/time.h> #include <odp/api/plat/time_inlines.h> +#include <odp_parse_internal.h> #include <odp_packet_io_internal.h> #include <odp_packet_io_stats.h> #include <odp_ethtool_stats.h> @@ -822,11 +823,14 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, odp_packet_t pkt; odp_pool_t pool = pkt_priv(pktio_entry)->pool; odp_packet_hdr_t *pkt_hdr; - odp_packet_hdr_t parsed_hdr; int i; int num; uint32_t max_len; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; + int num_rx = 0; + const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; /* Allocate maximum sized packets */ max_len = pkt_priv(pktio_entry)->mtu; @@ -837,45 +841,50 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, for (i = 0; i < num; i++) { netmap_slot_t slot; uint16_t len; + const uint8_t *buf; + uint64_t l4_part_sum = 0; slot = slot_tbl[i]; len = slot.len; + buf = (const uint8_t *)slot.buf; odp_prefetch(slot.buf); - if (pktio_cls_enabled(pktio_entry)) { - if (_odp_cls_classify_packet(pktio_entry, - (const uint8_t *)slot.buf, len, - len, &pool, &parsed_hdr, true)) - goto fail; + pkt = pkt_tbl[num_rx]; + pkt_hdr = packet_hdr(pkt); + + if (layer) { + if (_odp_packet_parse_common(&pkt_hdr->p, buf, len, len, + layer, chksums, &l4_part_sum, opt) < 0) + continue; + + if (pktio_cls_enabled(pktio_entry)) { + if (_odp_cls_classify_packet(pktio_entry, buf, &pool, + pkt_hdr)) + continue; + } } - pkt = pkt_tbl[i]; - pkt_hdr = packet_hdr(pkt); pull_tail(pkt_hdr, max_len - len); if (frame_offset) pull_head(pkt_hdr, frame_offset); if (odp_packet_copy_from_mem(pkt, 0, len, slot.buf) != 0) - goto fail; + break; pkt_hdr->input = pktio_entry->s.handle; - if (pktio_cls_enabled(pktio_entry)) - _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr); - else - _odp_packet_parse_layer(pkt_hdr, - pktio_entry->s.config.parser.layer, - pktio_entry->s.in_chksums); + if (layer >= ODP_PROTO_LAYER_L4) + _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); packet_set_ts(pkt_hdr, ts); + num_rx++; } - return i; + if (num_rx < num) + odp_packet_free_multi(&pkt_tbl[num_rx], num - num_rx); -fail: - odp_packet_free_multi(&pkt_tbl[i], num - i); - return i; + return num_rx; } static inline int netmap_recv_desc(pktio_entry_t *pktio_entry, @@ -1042,11 +1051,11 @@ static int netmap_recv_tmo(pktio_entry_t *pktio_entry, int index, } static int netmap_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[], - int num_q, odp_packet_t pkt_table[], int num, - unsigned *from, uint64_t usecs) + uint32_t num_q, odp_packet_t pkt_table[], int num, + uint32_t *from, uint64_t usecs) { struct timeval timeout; - int i; + uint32_t i; int ret; int maxfd = -1, maxfd2; fd_set readfds; diff --git a/platform/linux-generic/pktio/null.c b/platform/linux-generic/pktio/null.c index 90e113ec6..a6498da8c 100644 --- a/platform/linux-generic/pktio/null.c +++ b/platform/linux-generic/pktio/null.c @@ -71,9 +71,9 @@ static int null_recv_tmo(pktio_entry_t *pktio_entry ODP_UNUSED, } static int null_recv_mq_tmo(pktio_entry_t *pktio_entry[] ODP_UNUSED, - int index[] ODP_UNUSED, int num_q ODP_UNUSED, + int index[] ODP_UNUSED, uint32_t num_q ODP_UNUSED, odp_packet_t pkt_table[] ODP_UNUSED, - int num ODP_UNUSED, unsigned *from ODP_UNUSED, + int num ODP_UNUSED, uint32_t *from ODP_UNUSED, uint64_t usecs) { struct timeval timeout; diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c index 7290626c2..af94ffa72 100644 --- a/platform/linux-generic/pktio/pcap.c +++ b/platform/linux-generic/pktio/pcap.c @@ -46,6 +46,7 @@ #include <odp/api/plat/packet_inlines.h> +#include <odp_parse_internal.h> #include <odp_classification_internal.h> #include <odp_debug_internal.h> #include <odp_global_data.h> @@ -248,7 +249,12 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pkt_pcap_t *pcap = pkt_priv(pktio_entry); odp_time_t ts_val; odp_time_t *ts = NULL; + int packets = 0, errors = 0; + uint32_t octets = 0; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; + const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; odp_ticketlock_lock(&pktio_entry->s.rxl); @@ -256,8 +262,7 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_ticketlock_unlock(&pktio_entry->s.rxl); return 0; } - if (pktio_entry->s.config.pktin.bit.ts_all || - pktio_entry->s.config.pktin.bit.ts_ptp) + if (opt.bit.ts_all || opt.bit.ts_ptp) ts = &ts_val; for (i = 0; i < num; ) { @@ -291,42 +296,64 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, break; } - if (pktio_cls_enabled(pktio_entry)) { - odp_packet_t new_pkt; + if (layer) { + uint64_t l4_part_sum = 0; - ret = _odp_cls_classify_packet(pktio_entry, data, - pkt_len, pkt_len, - &new_pool, pkt_hdr, true); - if (ret) { + ret = _odp_packet_parse_common(&pkt_hdr->p, data, pkt_len, + pkt_len, layer, chksums, + &l4_part_sum, opt); + if (ret) + errors++; + + if (ret < 0) { odp_packet_free(pkt); continue; } - if (new_pool != pcap->pool) { - new_pkt = odp_packet_copy(pkt, new_pool); - odp_packet_free(pkt); + if (pktio_cls_enabled(pktio_entry)) { + odp_packet_t new_pkt; - if (odp_unlikely(new_pkt == ODP_PACKET_INVALID)) + ret = _odp_cls_classify_packet(pktio_entry, data, + &new_pool, pkt_hdr); + if (ret) { + odp_packet_free(pkt); continue; + } + if (new_pool != pcap->pool) { + new_pkt = odp_packet_copy(pkt, new_pool); + + odp_packet_free(pkt); + + if (odp_unlikely(new_pkt == ODP_PACKET_INVALID)) { + pktio_entry->s.stats.in_discards++; + continue; + } - pkt = new_pkt; - pkt_hdr = packet_hdr(new_pkt); + pkt = new_pkt; + pkt_hdr = packet_hdr(new_pkt); + } } - } else { - _odp_packet_parse_layer(pkt_hdr, - pktio_entry->s.config.parser.layer, - pktio_entry->s.in_chksums); + + if (layer >= ODP_PROTO_LAYER_L4) + _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); } - pktio_entry->s.stats.in_octets += pkt_hdr->frame_len; packet_set_ts(pkt_hdr, ts); pkt_hdr->input = pktio_entry->s.handle; + if (!pkt_hdr->p.flags.all.error) { + octets += pkt_len; + packets++; + } + pkts[i] = pkt; i++; } - pktio_entry->s.stats.in_packets += i; + + pktio_entry->s.stats.in_octets += octets; + pktio_entry->s.stats.in_packets += packets; + pktio_entry->s.stats.in_errors += errors; odp_ticketlock_unlock(&pktio_entry->s.rxl); @@ -441,6 +468,8 @@ static int pcapif_capability(pktio_entry_t *pktio_entry ODP_UNUSED, capa->stats.pktio.counter.in_octets = 1; capa->stats.pktio.counter.in_packets = 1; + capa->stats.pktio.counter.in_discards = 1; + capa->stats.pktio.counter.in_errors = 1; capa->stats.pktio.counter.out_octets = 1; capa->stats.pktio.counter.out_packets = 1; diff --git a/platform/linux-generic/pktio/pktio_common.c b/platform/linux-generic/pktio/pktio_common.c index d2f4d7219..1f3cf951e 100644 --- a/platform/linux-generic/pktio/pktio_common.c +++ b/platform/linux-generic/pktio/pktio_common.c @@ -11,13 +11,13 @@ static int sock_recv_mq_tmo_select(pktio_entry_t * const *entry, const int index[], - unsigned int num_q, unsigned int *from, + uint32_t num_q, uint32_t *from, odp_packet_t packets[], int num, uint64_t usecs, fd_set *readfds, int maxfd) { struct timeval timeout; - unsigned int i; + uint32_t i; int ret; for (i = 0; i < num_q; i++) { @@ -50,17 +50,17 @@ static int sock_recv_mq_tmo_select(pktio_entry_t * const *entry, } int _odp_sock_recv_mq_tmo_try_int_driven(const struct odp_pktin_queue_t queues[], - unsigned int num_q, unsigned int *from, + uint32_t num_q, uint32_t *from, odp_packet_t packets[], int num, uint64_t usecs, int *trial_successful) { - unsigned int i; + uint32_t i; pktio_entry_t *entry[num_q]; int index[num_q]; fd_set readfds; int maxfd = -1; - int (*impl)(pktio_entry_t *entry[], int index[], int num_q, - odp_packet_t packets[], int num, unsigned int *from, + int (*impl)(pktio_entry_t *entry[], int index[], uint32_t num_q, + odp_packet_t packets[], int num, uint32_t *from, uint64_t wait_usecs) = NULL; int impl_set = 0; diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c index 81c178b9d..9d1bbe545 100644 --- a/platform/linux-generic/pktio/socket.c +++ b/platform/linux-generic/pktio/socket.c @@ -15,12 +15,14 @@ #include <odp/api/ticketlock.h> #include <odp_socket_common.h> +#include <odp_parse_internal.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> #include <odp_packet_io_stats.h> #include <odp_debug_internal.h> #include <odp_errno_define.h> #include <odp_classification_internal.h> +#include <odp_macros_internal.h> #include <sys/socket.h> #include <stdio.h> @@ -233,6 +235,9 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, int i; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; uint32_t alloc_len = pkt_sock->mtu + frame_offset; + const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; memset(msgvec, 0, sizeof(msgvec)); @@ -249,8 +254,7 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, recv_msgs = recvmmsg(sockfd, msgvec, nb_pkts, MSG_DONTWAIT, NULL); odp_ticketlock_unlock(&pkt_sock->rx_lock); - if (pktio_entry->s.config.pktin.bit.ts_all || - pktio_entry->s.config.pktin.bit.ts_ptp) { + if (opt.bit.ts_all || opt.bit.ts_ptp) { ts_val = odp_time_global(); ts = &ts_val; } @@ -262,25 +266,48 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint16_t pkt_len = msgvec[i].msg_len; int ret; + uint64_t l4_part_sum = 0; if (odp_unlikely(msgvec[i].msg_hdr.msg_flags & MSG_TRUNC)) { odp_packet_free(pkt); ODP_DBG("dropped truncated packet\n"); continue; } - if (pktio_cls_enabled(pktio_entry)) { - uint16_t seg_len = pkt_len; - if (msgvec[i].msg_hdr.msg_iov->iov_len < pkt_len) - seg_len = msgvec[i].msg_hdr.msg_iov->iov_len; + ret = odp_packet_trunc_tail(&pkt, odp_packet_len(pkt) - pkt_len, + NULL, NULL); + if (ret < 0) { + ODP_ERR("trunc_tail failed"); + odp_packet_free(pkt); + continue; + } + + if (layer) { + uint8_t buf[PARSE_BYTES]; + uint16_t seg_len = msgvec[i].msg_hdr.msg_iov->iov_len; - if (_odp_cls_classify_packet(pktio_entry, base, pkt_len, - seg_len, &pool, pkt_hdr, - true)) { - ODP_ERR("_odp_cls_classify_packet failed"); + /* Make sure there is enough data for the packet + * parser in the case of a segmented packet. */ + if (odp_unlikely(seg_len < PARSE_BYTES && pkt_len > seg_len)) { + seg_len = MIN(pkt_len, PARSE_BYTES); + odp_packet_copy_to_mem(pkt, 0, seg_len, buf); + base = buf; + } + + if (_odp_packet_parse_common(&pkt_hdr->p, base, pkt_len, + seg_len, layer, chksums, + &l4_part_sum, opt) < 0) { odp_packet_free(pkt); continue; } + + if (pktio_cls_enabled(pktio_entry)) { + if (_odp_cls_classify_packet(pktio_entry, base, &pool, + pkt_hdr)) { + odp_packet_free(pkt); + continue; + } + } } /* Don't receive packets sent by ourselves */ @@ -290,20 +317,10 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, continue; } - ret = odp_packet_trunc_tail(&pkt, odp_packet_len(pkt) - pkt_len, - NULL, NULL); - if (ret < 0) { - ODP_ERR("trunk_tail failed"); - odp_packet_free(pkt); - continue; - } - pkt_hdr->input = pktio_entry->s.handle; - if (!pktio_cls_enabled(pktio_entry)) - _odp_packet_parse_layer(pkt_hdr, - pktio_entry->s.config.parser.layer, - pktio_entry->s.in_chksums); + if (layer >= ODP_PROTO_LAYER_L4) + _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); packet_set_ts(pkt_hdr, ts); @@ -359,11 +376,11 @@ static int sock_recv_tmo(pktio_entry_t *pktio_entry, int index, } static int sock_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[], - int num_q, odp_packet_t pkt_table[], int num, - unsigned *from, uint64_t usecs) + uint32_t num_q, odp_packet_t pkt_table[], int num, + uint32_t *from, uint64_t usecs) { struct timeval timeout; - int i; + uint32_t i; int ret; int maxfd = -1, maxfd2; fd_set readfds; diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index f84834610..7824b0e91 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -16,6 +16,7 @@ #include <odp/api/plat/packet_inlines.h> #include <odp_socket_common.h> +#include <odp_parse_internal.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> #include <odp_packet_io_stats.h> @@ -150,9 +151,11 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, odp_pool_t pool = pkt_sock->pool; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; uint16_t vlan_len = 0; + const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; - if (pktio_entry->s.config.pktin.bit.ts_all || - pktio_entry->s.config.pktin.bit.ts_ptp) + if (opt.bit.ts_all || opt.bit.ts_ptp) ts = &ts_val; ring = &pkt_sock->rx_ring; @@ -163,8 +166,8 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, struct tpacket2_hdr *tp_hdr; odp_packet_t pkt; odp_packet_hdr_t *hdr; - odp_packet_hdr_t parsed_hdr; int ret; + uint64_t l4_part_sum = 0; tp_hdr = (void *)next_ptr; @@ -212,18 +215,29 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, continue; } - if (pktio_cls_enabled(pktio_entry)) { - if (_odp_cls_classify_packet(pktio_entry, pkt_buf, pkt_len, - pkt_len, &pool, &parsed_hdr, - true)) { + hdr = packet_hdr(pkt); + + if (layer) { + if (_odp_packet_parse_common(&hdr->p, pkt_buf, pkt_len, + pkt_len, layer, chksums, + &l4_part_sum, opt) < 0) { odp_packet_free(pkt); tp_hdr->tp_status = TP_STATUS_KERNEL; frame_num = next_frame_num; continue; } + + if (pktio_cls_enabled(pktio_entry)) { + if (_odp_cls_classify_packet(pktio_entry, pkt_buf, + &pool, hdr)) { + odp_packet_free(pkt); + tp_hdr->tp_status = TP_STATUS_KERNEL; + frame_num = next_frame_num; + continue; + } + } } - hdr = packet_hdr(pkt); if (frame_offset) pull_head(hdr, frame_offset); @@ -276,12 +290,8 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, hdr->input = pktio_entry->s.handle; - if (pktio_cls_enabled(pktio_entry)) - _odp_packet_copy_cls_md(hdr, &parsed_hdr); - else - _odp_packet_parse_layer(hdr, - pktio_entry->s.config.parser.layer, - pktio_entry->s.in_chksums); + if (layer >= ODP_PROTO_LAYER_L4) + _odp_packet_l4_chksum(hdr, chksums, l4_part_sum); packet_set_ts(hdr, ts); @@ -727,11 +737,11 @@ static int sock_mmap_recv_tmo(pktio_entry_t *pktio_entry, int index, } static int sock_mmap_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[], - int num_q, odp_packet_t pkt_table[], int num, - unsigned *from, uint64_t usecs) + uint32_t num_q, odp_packet_t pkt_table[], int num, + uint32_t *from, uint64_t usecs) { struct timeval timeout; - int i; + uint32_t i; int ret; int maxfd = -1, maxfd2; fd_set readfds; diff --git a/platform/linux-generic/pktio/tap.c b/platform/linux-generic/pktio/tap.c index d298c8982..40c46b43c 100644 --- a/platform/linux-generic/pktio/tap.c +++ b/platform/linux-generic/pktio/tap.c @@ -38,6 +38,7 @@ #include <odp/api/plat/packet_inlines.h> +#include <odp_parse_internal.h> #include <odp_debug_internal.h> #include <odp_socket_common.h> #include <odp_packet_internal.h> @@ -286,14 +287,27 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data, odp_packet_hdr_t *pkt_hdr; odp_packet_hdr_t parsed_hdr; int num; + uint64_t l4_part_sum = 0; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; - - if (pktio_cls_enabled(pktio_entry)) { - if (_odp_cls_classify_packet(pktio_entry, data, len, len, - &pkt_priv(pktio_entry)->pool, - &parsed_hdr, true)) { + const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; + + if (layer) { + packet_parse_reset(&parsed_hdr, 1); + packet_set_len(&parsed_hdr, len); + if (_odp_packet_parse_common(&parsed_hdr.p, data, len, len, layer, + chksums, &l4_part_sum, opt) < 0) { return ODP_PACKET_INVALID; } + + if (pktio_cls_enabled(pktio_entry)) { + if (_odp_cls_classify_packet(pktio_entry, data, + &pkt_priv(pktio_entry)->pool, + &parsed_hdr)) { + return ODP_PACKET_INVALID; + } + } } num = _odp_packet_alloc_multi(pkt_priv(pktio_entry)->pool, @@ -312,12 +326,12 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data, return ODP_PACKET_INVALID; } - if (pktio_cls_enabled(pktio_entry)) + if (layer) { _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr); - else - _odp_packet_parse_layer(pkt_hdr, - pktio_entry->s.config.parser.layer, - pktio_entry->s.in_chksums); + + if (layer >= ODP_PROTO_LAYER_L4) + _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); + } packet_set_ts(pkt_hdr, ts); pkt_hdr->input = pktio_entry->s.handle; @@ -335,6 +349,7 @@ static int tap_pktio_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, uint8_t buf[mtu]; odp_time_t ts_val; odp_time_t *ts = NULL; + int num_rx = 0; odp_ticketlock_lock(&pktio_entry->s.rxl); @@ -355,14 +370,15 @@ static int tap_pktio_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, break; } - pkts[i] = pack_odp_pkt(pktio_entry, buf, retval, ts); - if (pkts[i] == ODP_PACKET_INVALID) + pkts[num_rx] = pack_odp_pkt(pktio_entry, buf, retval, ts); + if (pkts[num_rx] == ODP_PACKET_INVALID) break; + num_rx++; } odp_ticketlock_unlock(&pktio_entry->s.rxl); - return i; + return num_rx; } static int tap_pktio_send_lockless(pktio_entry_t *pktio_entry, |