aboutsummaryrefslogtreecommitdiff
path: root/platform/linux-dpdk
diff options
context:
space:
mode:
Diffstat (limited to 'platform/linux-dpdk')
-rw-r--r--platform/linux-dpdk/Makefile.am32
-rw-r--r--platform/linux-dpdk/include/odp_buffer_internal.h1
-rw-r--r--platform/linux-dpdk/include/odp_config_internal.h13
-rw-r--r--platform/linux-dpdk/include/odp_eventdev_internal.h2
-rw-r--r--platform/linux-dpdk/include/odp_queue_basic_internal.h3
-rw-r--r--platform/linux-dpdk/odp_crypto.c91
-rw-r--r--platform/linux-dpdk/odp_init.c1
-rw-r--r--platform/linux-dpdk/odp_packet.c106
-rw-r--r--platform/linux-dpdk/odp_pool.c2
-rw-r--r--platform/linux-dpdk/odp_queue_basic.c34
-rw-r--r--platform/linux-dpdk/odp_queue_eventdev.c2
-rw-r--r--platform/linux-dpdk/odp_schedule_if.c8
-rw-r--r--platform/linux-dpdk/odp_shared_memory.c21
-rw-r--r--platform/linux-dpdk/odp_system_info.c2
-rw-r--r--platform/linux-dpdk/odp_thread.c2
15 files changed, 266 insertions, 54 deletions
diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am
index 61b61f6d6..279aed0c1 100644
--- a/platform/linux-dpdk/Makefile.am
+++ b/platform/linux-dpdk/Makefile.am
@@ -242,12 +242,13 @@ __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \
arch/default/odp_cpu_cycles.c \
arch/default/odp_hash_crc32.c \
arch/arm/odp_sysinfo_parse.c
-odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \
- arch/default/odp/api/abi/cpu_time.h \
+odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \
arch/default/odp/api/abi/hash_crc32.h
if !ODP_ABI_COMPAT
odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \
arch/default/odp/api/abi/atomic_inlines.h \
+ arch/default/odp/api/abi/cpu_generic.h \
+ arch/arm/odp/api/abi/cpu_inlines.h \
arch/arm/odp/api/abi/cpu.h
endif
noinst_HEADERS += arch/arm/odp_atomic.h \
@@ -264,14 +265,15 @@ __LIB__libodp_dpdk_la_SOURCES += arch/aarch64/odp_atomic.c \
arch/aarch64/odp_global_time.c \
arch/default/odp_hash_crc32.c \
arch/aarch64/odp_sysinfo_parse.c
-odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \
- arch/aarch64/odp/api/abi/cpu_time.h \
+odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu_time.h \
arch/aarch64/odp/api/abi/hash_crc32.h
if !ODP_ABI_COMPAT
odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \
arch/aarch64/odp/api/abi/atomic_inlines.h \
arch/aarch64/odp/api/abi/atomic.h \
- arch/aarch64/odp/api/abi/cpu.h
+ arch/default/odp/api/abi/cpu_generic.h \
+ arch/aarch64/odp/api/abi/cpu_inlines.h \
+ arch/aarch64/odp/api/abi/cpu.h
endif
noinst_HEADERS += arch/aarch64/odp_atomic.h \
arch/aarch64/odp_cpu.h \
@@ -284,12 +286,13 @@ __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \
arch/default/odp_global_time.c \
arch/default/odp_hash_crc32.c \
arch/default/odp_sysinfo_parse.c
-odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \
- arch/default/odp/api/abi/cpu_time.h \
+odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \
arch/default/odp/api/abi/hash_crc32.h
if !ODP_ABI_COMPAT
odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \
arch/default/odp/api/abi/atomic_inlines.h \
+ arch/default/odp/api/abi/cpu_generic.h \
+ arch/default/odp/api/abi/cpu_inlines.h \
arch/default/odp/api/abi/cpu.h
endif
noinst_HEADERS += arch/default/odp_atomic.h \
@@ -298,16 +301,16 @@ noinst_HEADERS += arch/default/odp_atomic.h \
endif
if ARCH_IS_MIPS64
__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \
- arch/mips64/odp_cpu_cycles.c \
arch/default/odp_global_time.c \
arch/default/odp_hash_crc32.c \
arch/mips64/odp_sysinfo_parse.c
-odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \
- arch/default/odp/api/abi/cpu_time.h \
+odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \
arch/default/odp/api/abi/hash_crc32.h
if !ODP_ABI_COMPAT
odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \
arch/default/odp/api/abi/atomic_inlines.h \
+ arch/default/odp/api/abi/cpu_generic.h \
+ arch/mips64/odp/api/abi/cpu_inlines.h \
arch/mips64/odp/api/abi/cpu.h
endif
noinst_HEADERS += arch/default/odp_atomic.h \
@@ -320,12 +323,13 @@ __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \
arch/default/odp_global_time.c \
arch/default/odp_hash_crc32.c \
arch/powerpc/odp_sysinfo_parse.c
-odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \
- arch/default/odp/api/abi/cpu_time.h \
+odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \
arch/default/odp/api/abi/hash_crc32.h
if !ODP_ABI_COMPAT
odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \
arch/default/odp/api/abi/atomic_inlines.h \
+ arch/default/odp/api/abi/cpu_generic.h \
+ arch/default/odp/api/abi/cpu_inlines.h \
arch/powerpc/odp/api/abi/cpu.h
endif
noinst_HEADERS += arch/default/odp_atomic.h \
@@ -339,13 +343,13 @@ __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \
arch/x86/odp_global_time.c \
arch/default/odp_hash_crc32.c \
arch/x86/odp_sysinfo_parse.c
-odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_inlines.h \
- arch/x86/odp/api/abi/cpu_rdtsc.h \
+odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_rdtsc.h \
arch/x86/odp/api/abi/cpu_time.h \
arch/x86/odp/api/abi/hash_crc32.h
if !ODP_ABI_COMPAT
odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \
arch/default/odp/api/abi/atomic_inlines.h \
+ arch/x86/odp/api/abi/cpu_inlines.h \
arch/x86/odp/api/abi/cpu.h
endif
noinst_HEADERS += arch/x86/cpu_flags.h \
diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h b/platform/linux-dpdk/include/odp_buffer_internal.h
index 37e5d1b4f..e47030ea7 100644
--- a/platform/linux-dpdk/include/odp_buffer_internal.h
+++ b/platform/linux-dpdk/include/odp_buffer_internal.h
@@ -29,7 +29,6 @@ extern "C" {
#include <sys/types.h>
#include <odp/api/event.h>
#include <odp_forward_typedefs_internal.h>
-#include <odp_schedule_if.h>
#include <stddef.h>
/* DPDK */
diff --git a/platform/linux-dpdk/include/odp_config_internal.h b/platform/linux-dpdk/include/odp_config_internal.h
index 73d2304c9..6618d413d 100644
--- a/platform/linux-dpdk/include/odp_config_internal.h
+++ b/platform/linux-dpdk/include/odp_config_internal.h
@@ -124,11 +124,18 @@ extern "C" {
CONFIG_PACKET_HEADROOM - \
CONFIG_PACKET_TAILROOM)
-/* Maximum number of shared memory blocks.
+/*
+ * Number of shared memory blocks reserved for implementation internal use.
+ */
+#define CONFIG_INTERNAL_SHM_BLOCKS 20
+
+/*
+ * Maximum number of shared memory blocks.
*
- * This the number of separate SHM areas that can be reserved concurrently
+ * This is the number of separate SHM blocks that an application can reserve
+ * concurrently.
*/
-#define ODP_CONFIG_SHM_BLOCKS (ODP_CONFIG_POOLS + 48)
+#define CONFIG_SHM_BLOCKS 64
/*
* Maximum event burst size
diff --git a/platform/linux-dpdk/include/odp_eventdev_internal.h b/platform/linux-dpdk/include/odp_eventdev_internal.h
index 496a2238f..9e1083fd5 100644
--- a/platform/linux-dpdk/include/odp_eventdev_internal.h
+++ b/platform/linux-dpdk/include/odp_eventdev_internal.h
@@ -36,6 +36,8 @@ extern "C" {
#include <stdint.h>
+#define _ODP_SCHED_ID_EVENTDEV (_ODP_SCHED_ID_SCALABLE + 1)
+
#define RX_ADAPTER_INIT 0
#define RX_ADAPTER_STOPPED 1
#define RX_ADAPTER_RUNNING 2
diff --git a/platform/linux-dpdk/include/odp_queue_basic_internal.h b/platform/linux-dpdk/include/odp_queue_basic_internal.h
index c3ddaf334..1e542d973 100644
--- a/platform/linux-dpdk/include/odp_queue_basic_internal.h
+++ b/platform/linux-dpdk/include/odp_queue_basic_internal.h
@@ -116,6 +116,9 @@ int _odp_sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int num,
int update_status);
int _odp_sched_queue_empty(uint32_t queue_index);
+/* Functions by schedulers */
+int _odp_sched_basic_get_spread(uint32_t queue_index);
+
#ifdef __cplusplus
}
#endif
diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c
index 8ed561534..13e646230 100644
--- a/platform/linux-dpdk/odp_crypto.c
+++ b/platform/linux-dpdk/odp_crypto.c
@@ -64,8 +64,11 @@ typedef struct crypto_session_entry_s {
struct rte_cryptodev_sym_session *rte_session;
struct rte_crypto_sym_xform cipher_xform;
struct rte_crypto_sym_xform auth_xform;
+ struct {
+ unsigned int cdev_qpairs_shared:1;
+ unsigned int chained_bufs_ok:1;
+ } flags;
uint16_t cdev_nb_qpairs;
- odp_bool_t cdev_qpairs_shared;
uint8_t cdev_id;
uint8_t cipher_iv_data[MAX_IV_LENGTH];
uint8_t auth_iv_data[MAX_IV_LENGTH];
@@ -312,7 +315,7 @@ int _odp_crypto_init_global(void)
mem_size += (MAX_SESSIONS * sizeof(crypto_session_entry_t));
/* Allocate our globally shared memory */
- shm = odp_shm_reserve("_odp_crypto_glb", mem_size,
+ shm = odp_shm_reserve("_odp_crypto_global", mem_size,
ODP_CACHE_LINE_SIZE, 0);
if (shm != ODP_SHM_INVALID) {
global = odp_shm_addr(shm);
@@ -1260,6 +1263,34 @@ check_finish:
return -1;
}
+static int chained_bufs_ok(const odp_crypto_session_param_t *param,
+ uint8_t cdev_id)
+{
+ struct rte_cryptodev_info dev_info;
+ int chained_bufs_ok;
+
+ rte_cryptodev_info_get(cdev_id, &dev_info);
+ chained_bufs_ok = !!(dev_info.feature_flags & RTE_CRYPTODEV_FF_IN_PLACE_SGL);
+
+ /*
+ * Some crypto devices do not support chained buffers with all
+ * algorithms despite advertizing SG support in feature flags.
+ */
+
+ if (dev_info.driver_name &&
+ !strcmp(dev_info.driver_name, "crypto_aesni_gcm") &&
+ param->auth_alg == ODP_AUTH_ALG_AES_GMAC)
+ chained_bufs_ok = 0;
+
+ if (dev_info.driver_name &&
+ !strcmp(dev_info.driver_name, "crypto_openssl") &&
+ (param->cipher_alg == ODP_CIPHER_ALG_AES_GCM ||
+ param->auth_alg == ODP_AUTH_ALG_AES_GMAC))
+ chained_bufs_ok = 0;
+
+ return chained_bufs_ok;
+}
+
#if RTE_VERSION < RTE_VERSION_NUM(19, 8, 0, 0)
static int crypto_init_key(uint8_t **data, uint16_t *length,
odp_crypto_key_t *key, const char *type)
@@ -1482,6 +1513,7 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param,
param->auth_alg == ODP_AUTH_ALG_NULL) {
rte_session = NULL;
cdev_id = ~0;
+ session->flags.chained_bufs_ok = 1;
session->cdev_nb_qpairs = 0;
goto out_null;
} else if (param->cipher_alg == ODP_CIPHER_ALG_NULL) {
@@ -1524,8 +1556,12 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param,
goto err;
}
+ session->flags.chained_bufs_ok = chained_bufs_ok(param, cdev_id);
session->cdev_nb_qpairs = global->enabled_crypto_dev_qpairs[cdev_id];
- session->cdev_qpairs_shared = global->enabled_crypto_dev_qpairs_shared[cdev_id];
+ if (global->enabled_crypto_dev_qpairs_shared[cdev_id])
+ session->flags.cdev_qpairs_shared = 1;
+ else
+ session->flags.cdev_qpairs_shared = 0;
out_null:
session->rte_session = rte_session;
session->cdev_id = cdev_id;
@@ -1840,6 +1876,50 @@ static void crypto_fill_sym_param(crypto_session_entry_t *session,
op->sym->auth.data.length = param->auth_range.length;
}
+/*
+ * Attempt to change a multi segment packet to a single segment packet by
+ * reducing the headroom. Shift packet data toward the start of the first
+ * segment and trim the tail, hopefully getting rid of the tail segment.
+ *
+ * This fails if the packet data does not fit in the first segment with
+ * the new headroom. A temporary copy to a bigger buffer would be needed
+ * in that case.
+ *
+ * Do nothing for single segment packets.
+ *
+ * We assume that odp_crypto_operation() makes no promise to not shift
+ * packet data within the packet. If that is not the case, the shifting
+ * done here needs to be undone after the crypto operation.
+ *
+ */
+static int linearize_pkt(const crypto_session_entry_t *session, odp_packet_t pkt)
+{
+ const uint32_t new_headroom = CONFIG_PACKET_HEADROOM;
+ uint32_t headroom;
+ uint32_t len;
+ uint32_t shift;
+ int rc;
+
+ if (odp_likely(odp_packet_num_segs(pkt) == 1))
+ return 0;
+ if (session->flags.chained_bufs_ok)
+ return 0;
+
+ headroom = odp_packet_headroom(pkt);
+ if (odp_unlikely(new_headroom >= headroom))
+ return -1;
+
+ len = odp_packet_len(pkt);
+ shift = headroom - new_headroom;
+ odp_packet_push_head(pkt, shift);
+ odp_packet_move_data(pkt, 0, shift, len);
+ /* We rely on our trunc implementation to not change the handle */
+ rc = odp_packet_trunc_tail(&pkt, shift, NULL, NULL);
+ ODP_ASSERT(rc == 0);
+
+ return odp_packet_num_segs(pkt) != 1;
+}
+
static
int odp_crypto_int(odp_packet_t pkt_in,
odp_packet_t *pkt_out,
@@ -1896,6 +1976,9 @@ int odp_crypto_int(odp_packet_t pkt_in,
pkt_in = ODP_PACKET_INVALID;
}
+ if (linearize_pkt(session, out_pkt))
+ goto err;
+
rte_session = session->rte_session;
/* NULL rte_session means that it is a NULL-NULL operation.
* Just return new packet. */
@@ -1914,7 +1997,7 @@ int odp_crypto_int(odp_packet_t pkt_in,
int retry_count = 0;
int queue_pair;
int rc;
- odp_bool_t queue_pairs_shared = session->cdev_qpairs_shared;
+ odp_bool_t queue_pairs_shared = session->flags.cdev_qpairs_shared;
if (odp_unlikely(queue_pairs_shared))
queue_pair = odp_thread_id() % session->cdev_nb_qpairs;
diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c
index 914a48fad..45f00f8ac 100644
--- a/platform/linux-dpdk/odp_init.c
+++ b/platform/linux-dpdk/odp_init.c
@@ -74,6 +74,7 @@ static void disable_features(odp_global_data_ro_t *global_ro,
if (disable_ipsec && disable_crypto)
global_ro->disable.crypto = 1;
+ global_ro->disable.stash = init_param->not_used.feat.stash;
global_ro->disable.traffic_mngr = init_param->not_used.feat.tm;
global_ro->disable.compress = init_param->not_used.feat.compress;
}
diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c
index f5af3ce72..065a182a8 100644
--- a/platform/linux-dpdk/odp_packet.c
+++ b/platform/linux-dpdk/odp_packet.c
@@ -9,6 +9,7 @@
#include <odp/api/plat/packet_inlines.h>
#include <odp_packet_internal.h>
#include <odp_debug_internal.h>
+#include <odp_macros_internal.h>
#include <odp_chksum_internal.h>
#include <odp/api/hints.h>
#include <odp/api/byteorder.h>
@@ -365,8 +366,8 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, void **data_ptr,
}
/* Expand the original head segment*/
newhead->pkt_len += rte_pktmbuf_headroom(mb);
+ mb->data_len += rte_pktmbuf_headroom(mb);
mb->data_off = 0;
- mb->data_len = mb->buf_len;
_copy_head_metadata(newhead, mb);
mb = newhead;
*pkt = (odp_packet_t)newhead;
@@ -522,11 +523,18 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, void **tail_ptr,
uint32_t *tailroom)
{
struct rte_mbuf *mb = pkt_to_mbuf(*pkt);
+ struct rte_mbuf *last_mb = rte_pktmbuf_lastseg(mb);
if (odp_unlikely(len >= odp_packet_len(*pkt)))
return -1;
- if (rte_pktmbuf_trim(mb, len)) {
+ /*
+ * Trim only if the last segment does not become zero length.
+ */
+ if (odp_likely(len < last_mb->data_len)) {
+ if (odp_unlikely(rte_pktmbuf_trim(mb, len)))
+ return -1;
+ } else {
struct rte_mbuf *reverse[mb->nb_segs];
struct rte_mbuf *t = mb;
int i;
@@ -1311,7 +1319,15 @@ static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr,
return sum;
}
-/** Parser helper function for Ethernet packets */
+/*
+ * In the worst case we look at the Ethernet header, 8 bytes of LLC/SNAP
+ * header and two VLAN tags in the same packet.
+ */
+#define PARSE_ETH_BYTES (sizeof(_odp_ethhdr_t) + 8 + 2 * sizeof(_odp_vlanhdr_t))
+/** Parser helper function for Ethernet packets
+ *
+ * Requires up to PARSE_ETH_BYTES bytes of contiguous packet data.
+ */
static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr,
uint32_t *offset, uint32_t frame_len)
{
@@ -1328,7 +1344,7 @@ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr,
eth = (const _odp_ethhdr_t *)*parseptr;
/* Detect jumbo frames */
- if (odp_unlikely(frame_len > _ODP_ETH_LEN_MAX))
+ if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX))
input_flags.jumbo = 1;
/* Handle Ethernet broadcast/multicast addresses */
@@ -1386,14 +1402,27 @@ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr,
*parseptr += sizeof(_odp_vlanhdr_t);
}
+ /*
+ * The packet was too short for what we parsed. We just give up
+ * entirely without trying to parse what fits in the packet.
+ */
+ if (odp_unlikely(*offset > frame_len)) {
+ input_flags.all = 0;
+ input_flags.l2 = 1;
+ ethtype = 0;
+ }
+
error:
prs->input_flags.all |= input_flags.all;
return ethtype;
}
+#define PARSE_IPV4_BYTES (0xfU * 4) /* max IPv4 header length with options */
/**
* Parser helper function for IPv4
+ *
+ * Requires up to PARSE_IPV4_BYTES bytes of contiguous packet data.
*/
static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr,
uint32_t *offset, uint32_t frame_len,
@@ -1409,6 +1438,7 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr,
if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN ||
ver != 4 ||
+ sizeof(*ipv4) > frame_len - *offset ||
(l3_len > frame_len - *offset))) {
prs->flags.ip_err = 1;
return 0;
@@ -1451,8 +1481,15 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr,
return ipv4->proto;
}
+/*
+ * Peeks 2 bytes beyond IPv6 base header without length check if there
+ * are extension headers.
+ */
+#define PARSE_IPV6_BYTES (sizeof(_odp_ipv6hdr_t) + 2)
/**
* Parser helper function for IPv6
+ *
+ * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data.
*/
static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr,
uint32_t *offset, uint32_t frame_len,
@@ -1467,8 +1504,9 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr,
_ODP_IPV6HDR_LEN;
/* Basic sanity checks on IPv6 header */
- if ((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 ||
- l3_len > frame_len - *offset) {
+ if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 ||
+ sizeof(*ipv6) > frame_len - *offset ||
+ l3_len > frame_len - *offset)) {
prs->flags.ip_err = 1;
return 0;
}
@@ -1520,8 +1558,11 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr,
return ipv6->next_hdr;
}
+#define PARSE_TCP_BYTES (sizeof(_odp_tcphdr_t))
/**
* Parser helper function for TCP
+ *
+ * Requires PARSE_TCP_BYTES bytes of contiguous packet data.
*/
static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr,
uint16_t tcp_len,
@@ -1547,8 +1588,15 @@ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr,
*parseptr += len;
}
+/*
+ * In the worst case we look at the UDP header and 4 bytes of the UDP
+ * payload (the non-ESP marker to distinguish IKE packets from ESP packets).
+ */
+#define PARSE_UDP_BYTES (sizeof(_odp_udphdr_t) + 4)
/**
* Parser helper function for UDP
+ *
+ * Requires PARSE_UDP_BYTES bytes of contiguous packet data.
*/
static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr,
odp_proto_chksums_t chksums,
@@ -1593,8 +1641,11 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr,
*parseptr += sizeof(_odp_udphdr_t);
}
+#define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t))
/**
* Parser helper function for SCTP
+ *
+ * Requires PARSE_SCTP_BYTES bytes of contiguous packet data.
*/
static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr,
uint16_t sctp_len,
@@ -1621,6 +1672,10 @@ static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr,
*parseptr += sizeof(_odp_sctphdr_t);
}
+#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c)))
+#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \
+ MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES))
+/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */
static inline
int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr,
uint32_t offset,
@@ -1984,6 +2039,8 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
if (chksums.chksum.sctp &&
pkt_hdr->p.input_flags.sctp &&
!pkt_hdr->p.input_flags.ipfrag) {
+ uint32_t seg_len = 0;
+ _odp_sctphdr_t hdr_copy;
uint32_t sum = ~packet_sum_crc32c(pkt_hdr,
pkt_hdr->p.l4_offset +
_ODP_SCTPHDR_LEN,
@@ -1993,8 +2050,14 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
l4_part_sum);
_odp_sctphdr_t *sctp = odp_packet_offset(packet_handle(pkt_hdr),
pkt_hdr->p.l4_offset,
- NULL, NULL);
+ &seg_len, NULL);
+ if (odp_unlikely(seg_len < sizeof(*sctp))) {
+ odp_packet_t pkt = packet_handle(pkt_hdr);
+ sctp = &hdr_copy;
+ odp_packet_copy_to_mem(pkt, pkt_hdr->p.l4_offset,
+ sizeof(*sctp), sctp);
+ }
pkt_hdr->p.input_flags.l4_chksum_done = 1;
if (sum != sctp->chksum) {
pkt_hdr->p.flags.l4_chksum_err = 1;
@@ -2051,12 +2114,14 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
const uint8_t *data;
uint32_t seg_len;
- uint32_t len = odp_packet_len(pkt);
+ uint32_t packet_len = odp_packet_len(pkt);
odp_proto_t proto = param->proto;
odp_proto_layer_t layer = param->last_layer;
int ret;
uint16_t ethtype;
uint64_t l4_part_sum = 0;
+ const uint32_t min_seglen = PARSE_ETH_BYTES + PARSE_L3_L4_BYTES;
+ uint8_t buf[min_seglen];
if (proto == ODP_PROTO_NONE || layer == ODP_PROTO_LAYER_NONE)
return -1;
@@ -2066,6 +2131,20 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
if (data == NULL)
return -1;
+ /*
+ * We must not have a packet segment boundary within the parsed
+ * packet data range. Copy enough data to a temporary buffer for
+ * parsing if necessary.
+ */
+ if (odp_unlikely(pkt_hdr->buf_hdr.mb.nb_segs > 1) &&
+ odp_unlikely(seg_len < min_seglen)) {
+ seg_len = min_seglen;
+ if (seg_len > packet_len - offset)
+ seg_len = packet_len - offset;
+ odp_packet_copy_to_mem(pkt, offset, seg_len, buf);
+ data = buf;
+ }
+
/* Reset parser flags, keep other flags */
packet_parse_reset(pkt_hdr, 0);
@@ -2073,7 +2152,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
/* Assume valid L2 header, no CRC/FCS check in SW */
pkt_hdr->p.l2_offset = offset;
- ethtype = parse_eth(&pkt_hdr->p, &data, &offset, len);
+ ethtype = parse_eth(&pkt_hdr->p, &data, &offset, packet_len);
} else if (proto == ODP_PROTO_IPV4) {
ethtype = _ODP_ETHTYPE_IPV4;
} else if (proto == ODP_PROTO_IPV6) {
@@ -2083,7 +2162,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
}
ret = packet_parse_common_l3_l4(&pkt_hdr->p, data, offset,
- len, seg_len,
+ packet_len, seg_len,
layer, ethtype,
param->chksums,
&l4_part_sum);
@@ -2404,6 +2483,13 @@ odp_packet_reass_status_t odp_packet_reass_status(odp_packet_t pkt)
return ODP_PACKET_REASS_NONE;
}
+int odp_packet_reass_info(odp_packet_t pkt, odp_packet_reass_info_t *info)
+{
+ (void)pkt;
+ (void)info;
+ return -1;
+}
+
int odp_packet_reass_partial_state(odp_packet_t pkt, odp_packet_t frags[],
odp_packet_reass_partial_state_t *res)
{
diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c
index 4b5eb3206..4417ca4ee 100644
--- a/platform/linux-dpdk/odp_pool.c
+++ b/platform/linux-dpdk/odp_pool.c
@@ -153,7 +153,7 @@ int _odp_pool_init_global(void)
uint32_t i;
odp_shm_t shm;
- shm = odp_shm_reserve("_odp_pool_glb", sizeof(pool_global_t),
+ shm = odp_shm_reserve("_odp_pool_global", sizeof(pool_global_t),
ODP_CACHE_LINE_SIZE, 0);
_odp_pool_glb = odp_shm_addr(shm);
diff --git a/platform/linux-dpdk/odp_queue_basic.c b/platform/linux-dpdk/odp_queue_basic.c
index 7145fb2ce..33e0ba6bc 100644
--- a/platform/linux-dpdk/odp_queue_basic.c
+++ b/platform/linux-dpdk/odp_queue_basic.c
@@ -130,7 +130,7 @@ static int queue_init_global(void)
_odp_queue_inline_offset.context = offsetof(queue_entry_t,
s.param.context);
- shm = odp_shm_reserve("_odp_queue_gbl",
+ shm = odp_shm_reserve("_odp_queue_basic_global",
sizeof(queue_global_t),
sizeof(queue_entry_t), 0);
@@ -634,7 +634,8 @@ static void queue_print(odp_queue_t handle)
odp_pktio_info_t pktio_info;
queue_entry_t *queue;
uint32_t queue_id;
- int status;
+ int status, prio;
+ int max_prio = odp_schedule_max_prio();
queue_id = queue_to_index(handle);
@@ -658,7 +659,7 @@ static void queue_print(odp_queue_t handle)
ODP_PRINT("\nQueue info\n");
ODP_PRINT("----------\n");
ODP_PRINT(" handle %p\n", queue->s.handle);
- ODP_PRINT(" index %" PRIu32 "\n", queue->s.index);
+ ODP_PRINT(" index %" PRIu32 "\n", queue_id);
ODP_PRINT(" name %s\n", queue->s.name);
ODP_PRINT(" enq mode %s\n",
queue->s.param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" :
@@ -686,8 +687,11 @@ static void queue_print(odp_queue_t handle)
"ODP_SCHED_SYNC_ATOMIC" :
(queue->s.param.sched.sync == ODP_SCHED_SYNC_ORDERED ?
"ODP_SCHED_SYNC_ORDERED" : "unknown")));
- ODP_PRINT(" priority %d\n", queue->s.param.sched.prio);
- ODP_PRINT(" group %d\n", queue->s.param.sched.group);
+ prio = queue->s.param.sched.prio;
+ ODP_PRINT(" priority %i (%i in API)\n", max_prio - prio, prio);
+ ODP_PRINT(" group %i\n", queue->s.param.sched.group);
+ if (_odp_sched_id == _ODP_SCHED_ID_BASIC)
+ ODP_PRINT(" spread %i\n", _odp_sched_basic_get_spread(queue_id));
}
if (queue->s.pktin.pktio != ODP_PKTIO_INVALID) {
if (!odp_pktio_info(queue->s.pktin.pktio, &pktio_info))
@@ -744,11 +748,18 @@ static void queue_print_all(void)
char type_c, enq_c, deq_c, order_c, sync_c;
const int col_width = 24;
int prio = 0;
+ int spr = 0;
odp_schedule_sync_t sync = ODP_SCHED_SYNC_PARALLEL;
+ odp_schedule_group_t grp = ODP_SCHED_GROUP_INVALID;
ODP_PRINT("\nList of all queues\n");
ODP_PRINT("------------------\n");
- ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio\n", col_width, "name");
+ ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio grp",
+ col_width, "name");
+ if (_odp_sched_id == _ODP_SCHED_ID_BASIC)
+ ODP_PRINT(" spr\n");
+ else
+ ODP_PRINT("\n");
for (i = 0; i < CONFIG_MAX_QUEUES; i++) {
queue_entry_t *queue = qentry_from_index(i);
@@ -777,7 +788,10 @@ static void queue_print_all(void)
len = ring_st_length(queue->s.ring_st);
max_len = ring_st_max_length(queue->s.ring_st);
prio = queue->s.param.sched.prio;
+ grp = queue->s.param.sched.group;
sync = queue->s.param.sched.sync;
+ if (_odp_sched_id == _ODP_SCHED_ID_BASIC)
+ spr = _odp_sched_basic_get_spread(index);
} else {
len = ring_mpmc_length(queue->s.ring_mpmc);
max_len = ring_mpmc_max_length(queue->s.ring_mpmc);
@@ -811,7 +825,13 @@ static void queue_print_all(void)
if (type == ODP_QUEUE_TYPE_SCHED) {
sync_c = (sync == ODP_SCHED_SYNC_PARALLEL) ? 'P' :
((sync == ODP_SCHED_SYNC_ATOMIC) ? 'A' : 'O');
- ODP_PRINT(" %c %4i", sync_c, prio);
+ /* Print prio level matching odp_schedule_print() output */
+ prio = odp_schedule_max_prio() - prio;
+
+ ODP_PRINT(" %c %4i %3i", sync_c, prio, grp);
+
+ if (_odp_sched_id == _ODP_SCHED_ID_BASIC)
+ ODP_PRINT(" %3i", spr);
}
ODP_PRINT("\n");
diff --git a/platform/linux-dpdk/odp_queue_eventdev.c b/platform/linux-dpdk/odp_queue_eventdev.c
index 96baffa6f..0960c456e 100644
--- a/platform/linux-dpdk/odp_queue_eventdev.c
+++ b/platform/linux-dpdk/odp_queue_eventdev.c
@@ -528,7 +528,7 @@ static int queue_init_global(void)
_odp_queue_inline_offset.context = offsetof(queue_entry_t,
s.param.context);
- shm = odp_shm_reserve("_odp_eventdev_gbl",
+ shm = odp_shm_reserve("_odp_queue_eventdev_global",
sizeof(eventdev_global_t),
ODP_CACHE_LINE_SIZE, 0);
diff --git a/platform/linux-dpdk/odp_schedule_if.c b/platform/linux-dpdk/odp_schedule_if.c
index 29d38b1fd..21f5fc85b 100644
--- a/platform/linux-dpdk/odp_schedule_if.c
+++ b/platform/linux-dpdk/odp_schedule_if.c
@@ -1,4 +1,5 @@
/* Copyright (c) 2018, Linaro Limited
+ * Copyright (c) 2021, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -11,6 +12,9 @@
#include <odp_debug_internal.h>
#include <odp_global_data.h>
+/* Required for _ODP_SCHED_ID_EVENTDEV */
+#include <odp_eventdev_internal.h>
+
#include <stdlib.h>
#include <string.h>
@@ -25,6 +29,7 @@ extern const schedule_api_t _odp_schedule_eventdev_api;
const schedule_fn_t *_odp_sched_fn;
const schedule_api_t *_odp_sched_api;
+int _odp_sched_id;
uint64_t odp_schedule_wait_time(uint64_t ns)
{
@@ -216,12 +221,15 @@ int _odp_schedule_init_global(void)
ODP_PRINT("Using scheduler '%s'\n", sched);
if (!strcmp(sched, "basic")) {
+ _odp_sched_id = _ODP_SCHED_ID_BASIC;
_odp_sched_fn = &_odp_schedule_basic_fn;
_odp_sched_api = &_odp_schedule_basic_api;
} else if (!strcmp(sched, "sp")) {
+ _odp_sched_id = _ODP_SCHED_ID_SP;
_odp_sched_fn = &_odp_schedule_sp_fn;
_odp_sched_api = &_odp_schedule_sp_api;
} else if (!strcmp(sched, "eventdev")) {
+ _odp_sched_id = _ODP_SCHED_ID_EVENTDEV;
_odp_sched_fn = &_odp_schedule_eventdev_fn;
_odp_sched_api = &_odp_schedule_eventdev_api;
} else {
diff --git a/platform/linux-dpdk/odp_shared_memory.c b/platform/linux-dpdk/odp_shared_memory.c
index 4b1432a2f..645bb8847 100644
--- a/platform/linux-dpdk/odp_shared_memory.c
+++ b/platform/linux-dpdk/odp_shared_memory.c
@@ -1,4 +1,5 @@
/* Copyright (c) 2017-2018, Linaro Limited
+ * Copyright (c) 2021, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -25,9 +26,7 @@
#define SHM_MAX_ALIGN (0x80000000)
#define SHM_BLOCK_NAME "%" PRIu64 "-%d-%s"
-
-ODP_STATIC_ASSERT(ODP_CONFIG_SHM_BLOCKS >= ODP_CONFIG_POOLS,
- "ODP_CONFIG_SHM_BLOCKS < ODP_CONFIG_POOLS");
+#define SHM_MAX_NB_BLOCKS (CONFIG_INTERNAL_SHM_BLOCKS + CONFIG_SHM_BLOCKS)
ODP_STATIC_ASSERT(ODP_SHM_NAME_LEN >= RTE_MEMZONE_NAMESIZE,
"ODP_SHM_NAME_LEN < RTE_MEMZONE_NAMESIZE");
@@ -68,7 +67,7 @@ typedef struct {
*/
typedef struct {
odp_spinlock_t lock;
- shm_block_t block[ODP_CONFIG_SHM_BLOCKS];
+ shm_block_t block[SHM_MAX_NB_BLOCKS];
} shm_table_t;
static shm_table_t *shm_tbl;
@@ -80,7 +79,7 @@ static odp_bool_t mz_name_used(const char *name)
{
int idx;
- for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) {
+ for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) {
if (shm_tbl->block[idx].mz &&
strncmp(name, shm_tbl->block[idx].mz->name,
RTE_MEMZONE_NAMESIZE) == 0)
@@ -124,7 +123,7 @@ static int find_free_block(void)
{
int idx;
- for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) {
+ for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) {
if (shm_tbl->block[idx].mz == NULL)
return idx;
}
@@ -145,7 +144,7 @@ static inline odp_bool_t handle_is_valid(odp_shm_t shm)
{
int idx = handle_to_idx(shm);
- if (idx < 0 || idx >= ODP_CONFIG_SHM_BLOCKS ||
+ if (idx < 0 || idx >= SHM_MAX_NB_BLOCKS ||
shm_tbl->block[idx].mz == NULL) {
ODP_ERR("Invalid odp_shm_t handle: %" PRIu64 "\n",
odp_shm_to_u64(shm));
@@ -199,7 +198,7 @@ int _odp_shm_term_global(void)
}
/* Cleanup possibly non freed memory (and complain a bit) */
- for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) {
+ for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) {
block = &shm_tbl->block[idx];
if (block->mz) {
ODP_ERR("block '%s' was never freed (cleaning up...)\n",
@@ -224,7 +223,7 @@ int odp_shm_capability(odp_shm_capability_t *capa)
{
memset(capa, 0, sizeof(odp_shm_capability_t));
- capa->max_blocks = ODP_CONFIG_SHM_BLOCKS;
+ capa->max_blocks = CONFIG_SHM_BLOCKS;
capa->max_size = 0;
capa->max_align = SHM_MAX_ALIGN;
@@ -357,7 +356,7 @@ odp_shm_t odp_shm_lookup(const char *name)
odp_spinlock_lock(&shm_tbl->lock);
- for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) {
+ for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) {
if (shm_tbl->block[idx].mz &&
strncmp(name, shm_tbl->block[idx].name,
ODP_SHM_NAME_LEN) == 0) {
@@ -425,7 +424,7 @@ void odp_shm_print_all(void)
ODP_PRINT("\nShared memory blocks\n--------------------\n");
- for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) {
+ for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) {
block = &shm_tbl->block[idx];
if (block->mz == NULL)
continue;
diff --git a/platform/linux-dpdk/odp_system_info.c b/platform/linux-dpdk/odp_system_info.c
index af07f387a..30fcc140b 100644
--- a/platform/linux-dpdk/odp_system_info.c
+++ b/platform/linux-dpdk/odp_system_info.c
@@ -515,7 +515,7 @@ void odp_sys_config_print(void)
ODP_PRINT("CONFIG_PACKET_SEG_SIZE: %i\n", CONFIG_PACKET_SEG_SIZE);
ODP_PRINT("CONFIG_PACKET_SEG_LEN_MIN: %i\n", CONFIG_PACKET_SEG_LEN_MIN);
ODP_PRINT("CONFIG_PACKET_MAX_SEG_LEN: %i\n", CONFIG_PACKET_MAX_SEG_LEN);
- ODP_PRINT("ODP_CONFIG_SHM_BLOCKS: %i\n", ODP_CONFIG_SHM_BLOCKS);
+ ODP_PRINT("CONFIG_SHM_BLOCKS: %i\n", CONFIG_SHM_BLOCKS);
ODP_PRINT("CONFIG_BURST_SIZE: %i\n", CONFIG_BURST_SIZE);
ODP_PRINT("CONFIG_POOL_MAX_NUM: %i\n", CONFIG_POOL_MAX_NUM);
ODP_PRINT("\n");
diff --git a/platform/linux-dpdk/odp_thread.c b/platform/linux-dpdk/odp_thread.c
index 59394b3e4..7ab41cf72 100644
--- a/platform/linux-dpdk/odp_thread.c
+++ b/platform/linux-dpdk/odp_thread.c
@@ -76,7 +76,7 @@ int _odp_thread_init_global(void)
if (num_max > ODP_THREAD_COUNT_MAX)
num_max = ODP_THREAD_COUNT_MAX;
- shm = odp_shm_reserve("_odp_thread_globals",
+ shm = odp_shm_reserve("_odp_thread_global",
sizeof(thread_globals_t),
ODP_CACHE_LINE_SIZE, 0);
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-05 03:13:57 +0000