aboutsummaryrefslogtreecommitdiff
path: root/platform/linux-dpdk
diff options
context:
space:
mode:
Diffstat (limited to 'platform/linux-dpdk')
-rw-r--r--platform/linux-dpdk/Makefile.am1
l---------platform/linux-dpdk/include-abi/odp/api/abi/time_types.h1
-rw-r--r--platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h7
-rw-r--r--platform/linux-dpdk/include/odp/api/plat/packet_inlines.h2
-rw-r--r--platform/linux-dpdk/include/odp/api/plat/time_inlines.h7
-rw-r--r--platform/linux-dpdk/include/odp_event_internal.h3
-rw-r--r--platform/linux-dpdk/include/odp_packet_internal.h15
-rw-r--r--platform/linux-dpdk/include/odp_packet_io_internal.h1
-rw-r--r--platform/linux-dpdk/odp_init.c23
-rw-r--r--platform/linux-dpdk/odp_packet.c36
-rw-r--r--platform/linux-dpdk/odp_packet_dpdk.c128
-rw-r--r--platform/linux-dpdk/odp_pool.c60
-rw-r--r--platform/linux-dpdk/odp_timer.c93
13 files changed, 303 insertions, 74 deletions
diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am
index 76e205212..252b34fb2 100644
--- a/platform/linux-dpdk/Makefile.am
+++ b/platform/linux-dpdk/Makefile.am
@@ -114,6 +114,7 @@ odpapiabiarchinclude_HEADERS += \
include-abi/odp/api/abi/thrmask.h \
include-abi/odp/api/abi/ticketlock.h \
include-abi/odp/api/abi/time.h \
+ include-abi/odp/api/abi/time_types.h \
include-abi/odp/api/abi/timer.h \
include-abi/odp/api/abi/timer_types.h \
include-abi/odp/api/abi/traffic_mngr.h \
diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/time_types.h b/platform/linux-dpdk/include-abi/odp/api/abi/time_types.h
new file mode 120000
index 000000000..2f280c709
--- /dev/null
+++ b/platform/linux-dpdk/include-abi/odp/api/abi/time_types.h
@@ -0,0 +1 @@
+../../../../../linux-generic/include-abi/odp/api/abi/time_types.h \ No newline at end of file
diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h
index c6ede5d3a..5466ce58b 100644
--- a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h
+++ b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h
@@ -121,7 +121,7 @@ typedef union {
uint32_t all_flags;
struct {
- uint32_t reserved1: 6;
+ uint32_t reserved1: 5;
/*
* Init flags
@@ -140,6 +140,7 @@ typedef union {
uint32_t l4_chksum: 1; /* L4 chksum override */
uint32_t ts_set: 1; /* Set Tx timestamp */
uint32_t tx_compl: 1; /* Tx completion event requested */
+ uint32_t free_ctrl: 1; /* Don't free option */
uint32_t tx_aging: 1; /* Packet aging at Tx requested */
uint32_t shaper_len_adj: 8; /* Adjustment for traffic mgr */
@@ -157,8 +158,8 @@ typedef union {
/* Flag groups */
struct {
- uint32_t reserved2: 6;
- uint32_t other: 19; /* All other flags */
+ uint32_t reserved2: 5;
+ uint32_t other: 20; /* All other flags */
uint32_t error: 7; /* All error flags */
} all;
diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h
index 4ae53c7b3..7b2764a05 100644
--- a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h
+++ b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h
@@ -22,7 +22,7 @@ extern "C" {
#include <odp/api/hints.h>
#include <odp/api/packet_types.h>
#include <odp/api/pool_types.h>
-#include <odp/api/time.h>
+#include <odp/api/time_types.h>
#include <odp/api/plat/debug_inlines.h>
#include <odp/api/plat/event_validation_external.h>
diff --git a/platform/linux-dpdk/include/odp/api/plat/time_inlines.h b/platform/linux-dpdk/include/odp/api/plat/time_inlines.h
index 0ab2b37c0..4b1320300 100644
--- a/platform/linux-dpdk/include/odp/api/plat/time_inlines.h
+++ b/platform/linux-dpdk/include/odp/api/plat/time_inlines.h
@@ -1,5 +1,5 @@
/* Copyright (c) 2018, Linaro Limited
- * Copyright (c) 2020-2022, Nokia
+ * Copyright (c) 2020-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -8,13 +8,14 @@
#ifndef ODP_PLAT_TIME_INLINES_H_
#define ODP_PLAT_TIME_INLINES_H_
-#include <stdint.h>
-
#include <odp/api/align.h>
#include <odp/api/hints.h>
+#include <odp/api/time_types.h>
#include <odp/api/abi/cpu_time.h>
+#include <stdint.h>
+
/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
#define _ODP_TIMESPEC_SIZE 16
diff --git a/platform/linux-dpdk/include/odp_event_internal.h b/platform/linux-dpdk/include/odp_event_internal.h
index 6a707606a..cbe6d960b 100644
--- a/platform/linux-dpdk/include/odp_event_internal.h
+++ b/platform/linux-dpdk/include/odp_event_internal.h
@@ -46,6 +46,9 @@ typedef struct _odp_event_hdr_int_t {
/* Event type. Maybe different than pool type (crypto compl event) */
int8_t event_type;
+ /* Event subtype */
+ int8_t subtype;
+
/* Event flow id */
uint8_t flow_id;
diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h
index df4f2318e..37729b8f5 100644
--- a/platform/linux-dpdk/include/odp_packet_internal.h
+++ b/platform/linux-dpdk/include/odp_packet_internal.h
@@ -124,8 +124,10 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t {
/* Common internal header */
_odp_event_hdr_int_t event_hdr;
+ /* Parser metadata */
packet_parser_t p;
+ /* Input interface */
odp_pktio_t input;
/* Timestamp value */
@@ -162,9 +164,6 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t {
/* LSO profile index */
uint8_t lso_profile_idx;
- /* Event subtype */
- int8_t subtype;
-
union {
/* Result for crypto packet op */
odp_crypto_packet_result_t crypto_op_result;
@@ -215,7 +214,7 @@ static inline struct rte_mbuf *pkt_to_mbuf(odp_packet_t pkt)
static inline void packet_subtype_set(odp_packet_t pkt, int ev)
{
- packet_hdr(pkt)->subtype = ev;
+ packet_hdr(pkt)->event_hdr.subtype = ev;
}
/**
@@ -231,9 +230,7 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, odp_pktio_t input)
pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID;
pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID;
- if (odp_unlikely(pkt_hdr->subtype != ODP_EVENT_PACKET_BASIC))
- pkt_hdr->subtype = ODP_EVENT_PACKET_BASIC;
-
+ pkt_hdr->event_hdr.subtype = ODP_EVENT_PACKET_BASIC;
pkt_hdr->input = input;
}
@@ -277,10 +274,10 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr,
odp_packet_hdr_t *src_hdr,
odp_bool_t uarea_copy)
{
- const int8_t subtype = src_hdr->subtype;
+ const int8_t subtype = src_hdr->event_hdr.subtype;
dst_hdr->input = src_hdr->input;
- dst_hdr->subtype = subtype;
+ dst_hdr->event_hdr.subtype = subtype;
dst_hdr->dst_queue = src_hdr->dst_queue;
dst_hdr->cos = src_hdr->cos;
dst_hdr->cls_mark = src_hdr->cls_mark;
diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h
index 2efa52b67..ac22cff99 100644
--- a/platform/linux-dpdk/include/odp_packet_io_internal.h
+++ b/platform/linux-dpdk/include/odp_packet_io_internal.h
@@ -23,6 +23,7 @@ extern "C" {
#include <odp/api/packet_io.h>
#include <odp/api/spinlock.h>
#include <odp/api/ticketlock.h>
+#include <odp/api/time.h>
#include <odp/api/plat/packet_io_inlines.h>
diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c
index d35c63692..c7ae1ea47 100644
--- a/platform/linux-dpdk/odp_init.c
+++ b/platform/linux-dpdk/odp_init.c
@@ -26,6 +26,7 @@
#include <inttypes.h>
#include <rte_config.h>
+#include <rte_debug.h>
#include <rte_eal.h>
#include <rte_string_fns.h>
@@ -872,6 +873,28 @@ int odp_term_local(void)
return term_local(ALL_INIT);
}
+int odp_term_abnormal(odp_instance_t instance, uint64_t flags, void *data ODP_UNUSED)
+{
+ rte_dump_stack();
+
+ if (flags & ODP_TERM_FROM_SIGH)
+ /* Called from signal handler, not safe to terminate with local/global,
+ * return with failure as not able to perform all actions */
+ return -1;
+
+ if (odp_term_local() < 0) {
+ _ODP_ERR("ODP local terminate failed.\n");
+ return -2;
+ }
+
+ if (odp_term_global(instance) < 0) {
+ _ODP_ERR("ODP global terminate failed.\n");
+ return -3;
+ }
+
+ return 0;
+}
+
void odp_log_thread_fn_set(odp_log_func_t func)
{
_odp_this_thread->log_fn = func;
diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c
index 06c6c36ad..95149118e 100644
--- a/platform/linux-dpdk/odp_packet.c
+++ b/platform/linux-dpdk/odp_packet.c
@@ -64,7 +64,7 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = {
.timestamp = offsetof(odp_packet_hdr_t, timestamp),
.input_flags = offsetof(odp_packet_hdr_t, p.input_flags),
.flags = offsetof(odp_packet_hdr_t, p.flags),
- .subtype = offsetof(odp_packet_hdr_t, subtype),
+ .subtype = offsetof(odp_packet_hdr_t, event_hdr.subtype),
.cls_mark = offsetof(odp_packet_hdr_t, cls_mark),
.ipsec_ctx = offsetof(odp_packet_hdr_t, ipsec_ctx),
.crypto_op = offsetof(odp_packet_hdr_t, crypto_op_result),
@@ -1018,7 +1018,7 @@ void odp_packet_print(odp_packet_t pkt)
odp_packet_to_u64(pkt));
len += _odp_snprint(&str[len], n - len, " pool index %u\n", pool->pool_idx);
len += _odp_snprint(&str[len], n - len, " buf index %u\n", hdr->event_hdr.index);
- len += _odp_snprint(&str[len], n - len, " ev subtype %i\n", hdr->subtype);
+ len += _odp_snprint(&str[len], n - len, " ev subtype %i\n", hdr->event_hdr.subtype);
len += _odp_snprint(&str[len], n - len, " input_flags 0x%" PRIx64 "\n",
hdr->p.input_flags.all);
if (hdr->p.input_flags.all) {
@@ -1766,7 +1766,7 @@ int odp_packet_tx_compl_request(odp_packet_t pkt, const odp_packet_tx_compl_opt_
{
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
- pkt_hdr->p.flags.tx_compl = opt->mode == ODP_PACKET_TX_COMPL_ALL ? 1 : 0;
+ pkt_hdr->p.flags.tx_compl = opt->mode == ODP_PACKET_TX_COMPL_EVENT ? 1 : 0;
pkt_hdr->dst_queue = opt->queue;
return 0;
@@ -1801,6 +1801,34 @@ void *odp_packet_tx_compl_user_ptr(odp_packet_tx_compl_t tx_compl)
return (void *)(uintptr_t)data->user_ptr;
}
+int odp_packet_tx_compl_done(odp_pktio_t pktio, uint32_t compl_id)
+{
+ (void)pktio;
+ (void)compl_id;
+
+ return -1;
+}
+
+void odp_packet_free_ctrl_set(odp_packet_t pkt, odp_packet_free_ctrl_t ctrl)
+{
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+
+ if (ctrl == ODP_PACKET_FREE_CTRL_DONT_FREE)
+ pkt_hdr->p.flags.free_ctrl = 1;
+ else
+ pkt_hdr->p.flags.free_ctrl = 0;
+}
+
+odp_packet_free_ctrl_t odp_packet_free_ctrl(odp_packet_t pkt)
+{
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+
+ if (pkt_hdr->p.flags.free_ctrl)
+ return ODP_PACKET_FREE_CTRL_DONT_FREE;
+
+ return ODP_PACKET_FREE_CTRL_DISABLED;
+}
+
odp_packet_reass_status_t odp_packet_reass_status(odp_packet_t pkt)
{
(void)pkt;
@@ -1903,7 +1931,7 @@ odp_packet_t odp_packet_reassemble(odp_pool_t pool_hdl,
pkt_hdr->mb.data_off = headroom;
/* Reset metadata */
- pkt_hdr->subtype = ODP_EVENT_PACKET_BASIC;
+ pkt_hdr->event_hdr.subtype = ODP_EVENT_PACKET_BASIC;
pkt_hdr->input = ODP_PKTIO_INVALID;
packet_parse_reset(pkt_hdr, 1);
diff --git a/platform/linux-dpdk/odp_packet_dpdk.c b/platform/linux-dpdk/odp_packet_dpdk.c
index 238f733cd..c4fa3f0b7 100644
--- a/platform/linux-dpdk/odp_packet_dpdk.c
+++ b/platform/linux-dpdk/odp_packet_dpdk.c
@@ -127,6 +127,9 @@ typedef struct {
typedef struct ODP_ALIGNED_CACHE {
/* --- Fast path data --- */
+ /* Function for mbuf to ODP packet conversion */
+ int (*mbuf_to_pkt_fn)(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], uint16_t num);
+
/* DPDK port identifier */
uint16_t port_id;
struct {
@@ -187,6 +190,11 @@ extern void *pktio_entry_ptr[ODP_CONFIG_PKTIO_ENTRIES];
static uint32_t mtu_get_pkt_dpdk(pktio_entry_t *pktio_entry);
+static inline int input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], uint16_t num);
+
+static inline int input_pkts_minimal(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[],
+ uint16_t num);
+
uint16_t _odp_dpdk_pktio_port_id(pktio_entry_t *pktio_entry)
{
const pkt_dpdk_t *pkt_dpdk = pkt_priv(pktio_entry);
@@ -901,6 +909,12 @@ static int dpdk_start(pktio_entry_t *pktio_entry)
}
}
+ /* Use simpler function when packet parsing and classifying are not required */
+ if (pktio_entry->parse_layer == ODP_PROTO_LAYER_NONE)
+ pkt_dpdk->mbuf_to_pkt_fn = input_pkts_minimal;
+ else
+ pkt_dpdk->mbuf_to_pkt_fn = input_pkts;
+
/* Start device */
ret = rte_eth_dev_start(port_id);
if (ret < 0) {
@@ -929,19 +943,60 @@ static inline void prefetch_pkt(odp_packet_t pkt)
{
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
- odp_prefetch(&pkt_hdr->p);
+ odp_prefetch_store(&pkt_hdr->p);
}
-static inline int input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int num)
+/**
+ * Input packets when packet parsing and classifier are disabled
+ */
+static inline int input_pkts_minimal(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[],
+ uint16_t num)
{
uint16_t i;
- uint16_t num_pkts = 0;
- odp_pktin_config_opt_t pktin_cfg = pktio_entry->config.pktin;
- odp_pktio_t input = pktio_entry->handle;
odp_time_t ts_val;
odp_time_t *ts = NULL;
- uint16_t num_prefetch = RTE_MIN(num, NUM_RX_PREFETCH);
+ const uint8_t ts_ena = (pktio_entry->config.pktin.bit.ts_all ||
+ pktio_entry->config.pktin.bit.ts_ptp);
+ const odp_pktio_t input = pktio_entry->handle;
+ const uint16_t num_prefetch = RTE_MIN(num, NUM_RX_PREFETCH);
+
+ for (i = 0; i < num_prefetch; i++)
+ prefetch_pkt(pkt_table[i]);
+
+ if (ts_ena) {
+ ts_val = odp_time_global();
+ ts = &ts_val;
+ }
+
+ for (i = 0; i < num; ++i) {
+ odp_packet_t pkt = pkt_table[i];
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+
+ if (odp_likely(i + num_prefetch < num))
+ prefetch_pkt(pkt_table[i + num_prefetch]);
+
+ packet_init(pkt_hdr, input);
+
+ packet_set_ts(pkt_hdr, ts);
+ }
+
+ return num;
+}
+
+/**
+ * input packets when packet parsing is required
+ */
+static inline int input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], uint16_t num)
+{
+ uint16_t i;
+ uint16_t num_pkts = 0, num_cls = 0;
+ odp_time_t ts_val;
+ odp_time_t *ts = NULL;
+ const odp_pktin_config_opt_t pktin_cfg = pktio_entry->config.pktin;
+ const odp_pktio_t input = pktio_entry->handle;
const odp_proto_layer_t layer = pktio_entry->parse_layer;
+ const int cls_enabled = pktio_cls_enabled(pktio_entry);
+ const uint16_t num_prefetch = RTE_MIN(num, NUM_RX_PREFETCH);
for (i = 0; i < num_prefetch; i++)
prefetch_pkt(pkt_table[i]);
@@ -955,47 +1010,34 @@ static inline int input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[
odp_packet_t pkt = pkt_table[i];
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
struct rte_mbuf *mbuf = pkt_to_mbuf(pkt);
+ int ret;
if (odp_likely(i + num_prefetch < num))
prefetch_pkt(pkt_table[i + num_prefetch]);
packet_init(pkt_hdr, input);
- if (layer != ODP_PROTO_LAYER_NONE) {
- int ret;
-
- ret = _odp_dpdk_packet_parse_common(pkt_hdr,
- rte_pktmbuf_mtod(mbuf, uint8_t *),
- rte_pktmbuf_pkt_len(mbuf),
- rte_pktmbuf_data_len(mbuf),
- mbuf, layer, pktin_cfg);
- if (odp_unlikely(ret)) {
- odp_atomic_inc_u64(&pktio_entry->stats_extra.in_errors);
+ ret = _odp_dpdk_packet_parse_common(pkt_hdr,
+ rte_pktmbuf_mtod(mbuf, uint8_t *),
+ rte_pktmbuf_pkt_len(mbuf),
+ rte_pktmbuf_data_len(mbuf),
+ mbuf, layer, pktin_cfg);
+ if (odp_unlikely(ret)) {
+ odp_atomic_inc_u64(&pktio_entry->stats_extra.in_errors);
- if (ret < 0) {
- odp_packet_free(pkt);
- continue;
- }
+ if (ret < 0) {
+ odp_packet_free(pkt);
+ continue;
}
-
- if (odp_unlikely(num_pkts != i))
- pkt_table[num_pkts] = pkt;
}
+
packet_set_ts(pkt_hdr, ts);
odp_prefetch(rte_pktmbuf_mtod(mbuf, char *));
- num_pkts++;
- }
-
- if (pktio_cls_enabled(pktio_entry)) {
- int success = 0;
- for (i = 0; i < num_pkts; i++) {
- odp_packet_t pkt = pkt_table[i];
+ if (cls_enabled) {
odp_pool_t new_pool;
uint8_t *data = odp_packet_data(pkt);
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
- int ret;
ret = _odp_cls_classify_packet(pktio_entry, data, &new_pool, pkt_hdr);
if (odp_unlikely(ret)) {
@@ -1014,23 +1056,29 @@ static inline int input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[
odp_atomic_inc_u64(&pktio_entry->stats_extra.in_discards);
continue;
}
- pkt_table[i] = new_pkt;
pkt = new_pkt;
}
- if (success != i)
- pkt_table[success] = pkt;
- ++success;
+ /* Enqueue packets directly to classifier destination queue */
+ pkt_table[num_cls++] = pkt;
+ num_cls = _odp_cls_enq(pkt_table, num_cls, (i + 1 == num));
+ } else {
+ pkt_table[num_pkts++] = pkt;
}
- num_pkts = success;
}
+ /* Enqueue remaining classified packets */
+ if (odp_unlikely(num_cls))
+ _odp_cls_enq(pkt_table, num_cls, true);
+
return num_pkts;
}
int _odp_input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int num)
{
- return input_pkts(pktio_entry, pkt_table, num);
+ pkt_dpdk_t * const pkt_dpdk = pkt_priv(pktio_entry);
+
+ return pkt_dpdk->mbuf_to_pkt_fn(pktio_entry, pkt_table, num);
}
static int recv_pkt_dpdk(pktio_entry_t *pktio_entry, int index,
@@ -1071,10 +1119,10 @@ static int recv_pkt_dpdk(pktio_entry_t *pktio_entry, int index,
odp_ticketlock_unlock(&pkt_dpdk->rx_lock[index]);
/* Packets may also me received through eventdev, so don't add any
- * processing here. Instead, perform all processing in input_pkts()
+ * processing here. Instead, perform all processing in mbuf_to_pkt_fn()
* which is also called by eventdev. */
if (nb_rx)
- return input_pkts(pktio_entry, pkt_table, nb_rx);
+ return pkt_dpdk->mbuf_to_pkt_fn(pktio_entry, pkt_table, nb_rx);
return 0;
}
diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c
index 9dd1a5ca2..83d75dd6c 100644
--- a/platform/linux-dpdk/odp_pool.c
+++ b/platform/linux-dpdk/odp_pool.c
@@ -823,6 +823,24 @@ int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num)
return _odp_event_alloc_multi(pool, (_odp_event_hdr_t **)buf, num);
}
+static const char *get_short_type_str(odp_pool_type_t type)
+{
+ switch (type) {
+ case ODP_POOL_BUFFER:
+ return "B";
+ case ODP_POOL_PACKET:
+ return "P";
+ case ODP_POOL_TIMEOUT:
+ return "T";
+ case ODP_POOL_VECTOR:
+ return "V";
+ case ODP_POOL_DMA_COMPL:
+ return "D";
+ default:
+ return "-";
+ }
+}
+
void odp_pool_print(odp_pool_t pool_hdl)
{
pool_t *pool = _odp_pool_entry(pool_hdl);
@@ -837,8 +855,7 @@ void odp_pool_print_all(void)
uint32_t elt_size, elt_len = 0;
uint8_t type, ext;
const int col_width = 24;
- const char *name;
- char type_c;
+ const char *name, *type_c;
_ODP_PRINT("\nList of all pools\n");
_ODP_PRINT("-----------------\n");
@@ -868,12 +885,9 @@ void odp_pool_print_all(void)
if (type == ODP_POOL_BUFFER || type == ODP_POOL_PACKET)
elt_len = elt_size;
- type_c = (type == ODP_POOL_BUFFER) ? 'B' :
- (type == ODP_POOL_PACKET) ? 'P' :
- (type == ODP_POOL_TIMEOUT) ? 'T' :
- (type == ODP_POOL_VECTOR) ? 'V' : '-';
+ type_c = get_short_type_str(pool->type_2);
- _ODP_PRINT("%4u %-*s %c %6" PRIu64 " %6" PRIu32 " %6" PRIu32 " %8" PRIu32 " "
+ _ODP_PRINT("%4u %-*s %s %6" PRIu64 " %6" PRIu32 " %6" PRIu32 " %8" PRIu32 " "
"%" PRIu8 "\n", index, col_width, name, type_c, available, tot,
cache_size, elt_len, ext);
}
@@ -913,6 +927,7 @@ int odp_pool_info(odp_pool_t pool_hdl, odp_pool_info_t *info)
} else if (pool->type_2 == ODP_POOL_DMA_COMPL) {
info->dma_pool_param.num = pool->params.buf.num;
+ info->dma_pool_param.uarea_size = pool->params.buf.uarea_size;
info->dma_pool_param.cache_size = pool->params.buf.cache_size;
} else {
@@ -1004,6 +1019,37 @@ int odp_pool_stats(odp_pool_t pool_hdl, odp_pool_stats_t *stats)
return 0;
}
+int odp_pool_stats_selected(odp_pool_t pool_hdl, odp_pool_stats_selected_t *stats,
+ const odp_pool_stats_opt_t *opt)
+{
+ pool_t *pool;
+
+ if (odp_unlikely(pool_hdl == ODP_POOL_INVALID)) {
+ _ODP_ERR("Invalid pool handle\n");
+ return -1;
+ }
+ if (odp_unlikely(stats == NULL)) {
+ _ODP_ERR("Output buffer NULL\n");
+ return -1;
+ }
+ if (odp_unlikely(opt == NULL)) {
+ _ODP_ERR("Pool counters NULL\n");
+ return -1;
+ }
+
+ pool = _odp_pool_entry(pool_hdl);
+
+ if (odp_unlikely(opt->all & ~pool->params.stats.all)) {
+ _ODP_ERR("Trying to read disabled counter\n");
+ return -1;
+ }
+
+ if (opt->bit.available)
+ stats->available = rte_mempool_avail_count(pool->rte_mempool);
+
+ return 0;
+}
+
int odp_pool_stats_reset(odp_pool_t pool_hdl ODP_UNUSED)
{
return 0;
diff --git a/platform/linux-dpdk/odp_timer.c b/platform/linux-dpdk/odp_timer.c
index 1021a96cd..4056018ae 100644
--- a/platform/linux-dpdk/odp_timer.c
+++ b/platform/linux-dpdk/odp_timer.c
@@ -93,6 +93,9 @@ ODP_STATIC_ASSERT(MAX_TIMERS < MAX_TIMER_RING_SIZE,
/* Maximum number of periodic timers per pool */
#define MAX_PERIODIC_TIMERS 100
+/* Periodic tick fractional part accumulator size */
+#define ACC_SIZE (1ull << 32)
+
typedef struct {
odp_ticketlock_t lock;
uint64_t tick;
@@ -102,8 +105,13 @@ typedef struct {
struct timer_pool_s *timer_pool;
int state;
uint32_t timer_idx;
+
/* Period of periodic timer in ticks, includes PERIODIC_CANCELLED flag. */
uint64_t periodic_ticks;
+ /* Periodic ticks fractional part. */
+ uint32_t periodic_ticks_frac;
+ /* Periodic ticks fractional part accumulator. */
+ uint32_t periodic_ticks_frac_acc;
struct rte_timer rte_timer;
@@ -509,14 +517,26 @@ int odp_timer_res_capability(odp_timer_clk_src_t clk_src,
}
if (res_capa->min_tmo) {
- _ODP_ERR("Only res_ns or max_tmo based quaries supported\n");
+ _ODP_ERR("Only res_ns or max_tmo based queries supported\n");
return -1;
}
if (res_capa->res_ns || res_capa->res_hz) {
+ if (res_capa->res_ns && res_capa->res_ns < MAX_RES_NS) {
+ _ODP_DBG("Timeout resolution capability (res_ns) exceeded\n");
+ return -1;
+ }
+ if (res_capa->res_hz && res_capa->res_hz > MAX_RES_HZ) {
+ _ODP_DBG("Timeout resolution capability (res_hz) exceeded\n");
+ return -1;
+ }
res_capa->min_tmo = min_tmo;
res_capa->max_tmo = MAX_TMO_NS;
} else { /* max_tmo */
+ if (res_capa->max_tmo > MAX_TMO_NS) {
+ _ODP_DBG("Maximum relative timeout capability (max_tmo) exceeded\n");
+ return -1;
+ }
res_capa->min_tmo = min_tmo;
res_capa->res_ns = MAX_RES_NS;
res_capa->res_hz = MAX_RES_HZ;
@@ -787,6 +807,35 @@ uint64_t odp_timer_current_tick(odp_timer_pool_t tp)
return rte_get_timer_cycles();
}
+int odp_timer_sample_ticks(odp_timer_pool_t tp[], uint64_t tick[], uint64_t clk_count[], int num)
+{
+ uint64_t now;
+ int i;
+
+ if (num <= 0 || num > MAX_TIMER_POOLS) {
+ _ODP_ERR("Bad number of timer pools: %i\n", num);
+ return -1;
+ }
+
+ for (i = 0; i < num; i++) {
+ if (odp_unlikely(tp[i] == ODP_TIMER_POOL_INVALID)) {
+ _ODP_ERR("Invalid timer pool\n");
+ return -1;
+ }
+ }
+
+ now = rte_get_timer_cycles();
+
+ for (i = 0; i < num; i++) {
+ tick[i] = now;
+
+ if (clk_count)
+ clk_count[i] = 0;
+ }
+
+ return 0;
+}
+
int odp_timer_pool_info(odp_timer_pool_t tp,
odp_timer_pool_info_t *info)
{
@@ -1065,6 +1114,7 @@ int odp_timer_periodic_start(odp_timer_t timer_hdl,
timer_pool_t *tp = timer->timer_pool;
uint64_t multiplier = start_param->freq_multiplier;
double freq = multiplier * tp->base_freq;
+ double period_ns_dbl;
int absolute;
int ret;
@@ -1083,13 +1133,18 @@ int odp_timer_periodic_start(odp_timer_t timer_hdl,
return ODP_TIMER_FAIL;
}
- period_ns = (uint64_t)((double)ODP_TIME_SEC_IN_NS / freq);
+ period_ns_dbl = (double)ODP_TIME_SEC_IN_NS / freq;
+ period_ns = period_ns_dbl;
+
if (period_ns == 0) {
_ODP_ERR("Too high periodic timer frequency: %f\n", freq);
return ODP_TIMER_FAIL;
}
timer->periodic_ticks = odp_timer_ns_to_tick(timer_pool_to_hdl(tp), period_ns);
+ timer->periodic_ticks_frac = (period_ns_dbl - period_ns) * ACC_SIZE;
+ timer->periodic_ticks_frac_acc = 0;
+
first_tick = timer->periodic_ticks;
absolute = 0;
@@ -1113,7 +1168,7 @@ int odp_timer_periodic_start(odp_timer_t timer_hdl,
int odp_timer_periodic_ack(odp_timer_t timer_hdl, odp_event_t tmo_ev)
{
- uint64_t abs_tick;
+ uint64_t abs_tick, acc;
odp_timeout_t tmo = odp_timeout_from_event(tmo_ev);
timer_entry_t *timer = timer_from_hdl(timer_hdl);
odp_timeout_hdr_t *timeout_hdr;
@@ -1129,6 +1184,15 @@ int odp_timer_periodic_ack(odp_timer_t timer_hdl, odp_event_t tmo_ev)
if (odp_unlikely(abs_tick == PERIODIC_CANCELLED))
return 2;
+ acc = (uint64_t)timer->periodic_ticks_frac_acc + (uint64_t)timer->periodic_ticks_frac;
+
+ if (acc >= ACC_SIZE) {
+ abs_tick++;
+ acc -= ACC_SIZE;
+ }
+
+ timer->periodic_ticks_frac_acc = acc;
+
timeout_hdr = timeout_to_hdr(tmo);
abs_tick += timeout_hdr->expiration;
timeout_hdr->expiration = abs_tick;
@@ -1156,14 +1220,19 @@ int odp_timer_cancel(odp_timer_t timer_hdl, odp_event_t *tmo_ev)
odp_ticketlock_lock(&timer->lock);
if (odp_unlikely(timer->state < TICKING)) {
+ int state = timer->state;
+
odp_ticketlock_unlock(&timer->lock);
- return -1;
+
+ if (state == EXPIRED)
+ return ODP_TIMER_TOO_NEAR;
+ return ODP_TIMER_FAIL;
}
if (odp_unlikely(timer_global->ops.stop(&timer->rte_timer))) {
/* Another core runs timer callback function. */
odp_ticketlock_unlock(&timer->lock);
- return -1;
+ return ODP_TIMER_TOO_NEAR;
}
*tmo_ev = timer->tmo_event;
@@ -1171,7 +1240,7 @@ int odp_timer_cancel(odp_timer_t timer_hdl, odp_event_t *tmo_ev)
timer->state = NOT_TICKING;
odp_ticketlock_unlock(&timer->lock);
- return 0;
+ return ODP_TIMER_SUCCESS;
}
int odp_timer_periodic_cancel(odp_timer_t timer_hdl)
@@ -1228,8 +1297,14 @@ uint64_t odp_timeout_to_u64(odp_timeout_t tmo)
int odp_timeout_fresh(odp_timeout_t tmo)
{
+ timer_entry_t *timer;
odp_timeout_hdr_t *timeout_hdr = timeout_to_hdr(tmo);
- timer_entry_t *timer = timer_from_hdl(timeout_hdr->timer);
+
+ /* Timeout not connected to a timer */
+ if (odp_unlikely(timeout_hdr->timer == ODP_TIMER_INVALID))
+ return 0;
+
+ timer = timer_from_hdl(timeout_hdr->timer);
if (timer->timer_pool->periodic)
return timer->periodic_ticks != PERIODIC_CANCELLED;
@@ -1240,6 +1315,7 @@ int odp_timeout_fresh(odp_timeout_t tmo)
odp_timeout_t odp_timeout_alloc(odp_pool_t pool_hdl)
{
+ odp_timeout_hdr_t *timeout_hdr;
odp_event_t event;
pool_t *pool;
@@ -1253,6 +1329,9 @@ odp_timeout_t odp_timeout_alloc(odp_pool_t pool_hdl)
if (odp_unlikely(event == ODP_EVENT_INVALID))
return ODP_TIMEOUT_INVALID;
+ timeout_hdr = timeout_to_hdr(odp_timeout_from_event(event));
+ timeout_hdr->timer = ODP_TIMER_INVALID;
+
return odp_timeout_from_event(event);
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-17 00:45:56 +0000