diff options
177 files changed, 7811 insertions, 6067 deletions
diff --git a/.github/workflows/ci-pipeline-arm64.yml b/.github/workflows/ci-pipeline-arm64.yml index 6cc5e38a7..3f7304e91 100644 --- a/.github/workflows/ci-pipeline-arm64.yml +++ b/.github/workflows/ci-pipeline-arm64.yml @@ -180,8 +180,6 @@ jobs: Run_dpdk-20_11: if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }} runs-on: [self-hosted, ARM64] - env: - OS: ubuntu_20.04 steps: - uses: AutoModality/action-clean@v1.1.0 - uses: actions/checkout@v3 @@ -194,8 +192,6 @@ jobs: Run_dpdk-21_11: if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }} runs-on: [self-hosted, ARM64] - env: - OS: ubuntu_20.04 steps: - uses: AutoModality/action-clean@v1.1.0 - uses: actions/checkout@v3 diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index a88d0feb2..a8ad4914d 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -5,7 +5,7 @@ env: ARCH: x86_64 CC: gcc CONTAINER_NAMESPACE: ghcr.io/opendataplane/odp-docker-images - OS: ubuntu_18.04 + OS: ubuntu_20.04 jobs: Checkpatch: @@ -104,40 +104,32 @@ jobs: runs-on: ubuntu-20.04 env: ARCH: arm64 + CONF: "--enable-dpdk-shared" strategy: fail-fast: false matrix: cc: [gcc, clang] + conf: ['', '--enable-abi-compat', 'CFLAGS=-march=armv8.2-a', 'CFLAGS=-march=armv8-a+lse'] steps: - uses: actions/checkout@v3 + - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" + -e CONF="${CONF} ${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name config.log -exec cat {} \; - - name: Minimal - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" - -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - - - name: --enable-abi-compat - env: - CONF: "--enable-abi-compat" - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" - -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - - - name: CFLAGS="-march=armv8.2-a" - env: - CONF: "CFLAGS=-march=armv8.2-a" - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" - -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - - - name: CFLAGS="-march=armv8-a+lse" - env: - CONF: "CFLAGS=-march=armv8-a+lse" - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" - -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - - - name: Ubuntu 20.04 - env: - CONF: "--enable-dpdk-shared" - OS: ubuntu_20.04 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.compiler}}" + Build_arm64_u18: + runs-on: ubuntu-20.04 + env: + ARCH: arm64 + OS: ubuntu_18.04 + strategy: + fail-fast: false + matrix: + cc: [gcc, clang] + steps: + - uses: actions/checkout@v3 + - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - name: Failure log if: ${{ failure() }} @@ -147,6 +139,7 @@ jobs: runs-on: ubuntu-20.04 env: ARCH: ppc64el + CONF: "--enable-dpdk-shared" strategy: fail-fast: false matrix: @@ -154,7 +147,7 @@ jobs: steps: - uses: actions/checkout@v3 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" - -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh + -e CONF="${CONF} ${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - name: Failure log if: ${{ failure() }} run: find . -name config.log -exec cat {} \; @@ -163,6 +156,7 @@ jobs: runs-on: ubuntu-20.04 env: ARCH: i386 + OS: ubuntu_18.04 strategy: fail-fast: false matrix: @@ -192,14 +186,14 @@ jobs: if: ${{ failure() }} run: find . -name config.log -exec cat {} \; - Build_gcc_u20: + Build_gcc_u18: runs-on: ubuntu-20.04 env: - OS: ubuntu_20.04 + OS: ubuntu_18.04 strategy: fail-fast: false matrix: - cc_ver: [9] + cc_ver: [7, 8] conf: ['', '--enable-abi-compat'] steps: - uses: actions/checkout@v3 @@ -299,7 +293,7 @@ jobs: fail-fast: false matrix: cc: [gcc, clang] - os: ['ubuntu_20.04', 'ubuntu_22.04-openssl'] + os: ['ubuntu_18.04', 'ubuntu_22.04-openssl'] steps: - uses: actions/checkout@v3 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e ARCH="${ARCH}" @@ -351,8 +345,6 @@ jobs: Run_dpdk-20_11: runs-on: ubuntu-20.04 - env: - OS: ubuntu_20.04 steps: - uses: actions/checkout@v3 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" @@ -363,8 +355,6 @@ jobs: Run_dpdk-21_11: runs-on: ubuntu-20.04 - env: - OS: ubuntu_20.04 steps: - uses: actions/checkout@v3 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" @@ -375,8 +365,6 @@ jobs: Run_crypto: runs-on: ubuntu-20.04 - env: - OS: ubuntu_20.04 strategy: fail-fast: false matrix: diff --git a/Makefile.am b/Makefile.am index e7b88032b..663cacf6f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,7 +1,6 @@ ACLOCAL_AMFLAGS=-I m4 AM_DISTCHECK_CONFIGURE_FLAGS = --enable-user-guides \ - --enable-helper-linux \ - --with-testdir + --enable-helper-linux if PLATFORM_IS_LINUX_GENERIC PLATFORM_DIR = platform/linux-generic diff --git a/configure.ac b/configure.ac index 6bbb0ae03..3eaf6a8a2 100644 --- a/configure.ac +++ b/configure.ac @@ -429,6 +429,11 @@ AC_SUBST([ODP_LDFLAGS]) AC_SUBST([EXEEXT]) +AM_CONDITIONAL([enable_static], [test x$enable_static = xyes ]) +AM_CONDITIONAL([enable_shared], [test x$enable_shared = xyes ]) + +AM_CONDITIONAL([cross_compile], [test x$host != x -a x$host != x$build ]) + CC_VERSION=$($CC --version | head -n 1) AC_OUTPUT @@ -441,6 +446,7 @@ AC_MSG_RESULT([ Helper version: ${ODPH_VERSION} implementation_name: ${ODP_IMPLEMENTATION_NAME} + build: ${build} host: ${host} ARCH_DIR ${ARCH_DIR} ARCH_ABI ${ARCH_ABI} @@ -452,7 +458,6 @@ AC_MSG_RESULT([ libdir: ${libdir} includedir: ${includedir} testdir: ${testdir} - WITH_ARCH: ${WITH_ARCH} cc: ${CC} cc version: ${CC_VERSION} diff --git a/doc/implementers-guide/implementers-guide.adoc b/doc/implementers-guide/implementers-guide.adoc index 0b039630c..568760e3a 100644 --- a/doc/implementers-guide/implementers-guide.adoc +++ b/doc/implementers-guide/implementers-guide.adoc @@ -750,7 +750,7 @@ invalid parameters are passed to ODP APIs. This is done for performance reasons so that implementations don't need to insert extraneous parameter checking that would impact runtime performance in fast-path operations. While this is a reasonable trade off, it can complicate application debugging. -To address this, the ODP implementation makes use of the `ODP_ASSERT()` macro +To address this, the ODP implementation makes use of the `_ODP_ASSERT()` macro that by default disappears at compile time unless the `--enable-debug` configuration option was specified. Running with a debug build of ODP trades off performance for improved parameter/bounds checking to make application diff --git a/example/classifier/odp_classifier.c b/example/classifier/odp_classifier.c index 7cab9bad6..99bebf4d4 100644 --- a/example/classifier/odp_classifier.c +++ b/example/classifier/odp_classifier.c @@ -25,10 +25,20 @@ */ #define MAX_WORKERS (ODP_THREAD_COUNT_MAX - 1) +/** @def MAX_PKT_BURST + * @brief Maximum packet burst size + */ +#define MAX_PKT_BURST 64 + +/** @def DEF_PKT_BURST + * @brief Default packet burst size + */ +#define DEF_PKT_BURST 32 + /** @def SHM_PKT_POOL_SIZE - * @brief Size of the shared memory block + * @brief Packet pool size (number of packets) */ -#define SHM_PKT_POOL_SIZE (512*2048) +#define SHM_PKT_POOL_SIZE 10000 /** @def SHM_PKT_POOL_BUF_SIZE * @brief Buffer size of the packet pool buffer @@ -80,6 +90,8 @@ typedef struct { } ci_pass_counters; typedef struct { + odp_pktout_queue_t pktout[MAX_WORKERS]; + int num_pktout; global_statistics stats[MAX_PMR_COUNT]; ci_pass_counters ci_pass_rules[MAX_PMR_COUNT]; int policy_count; /**< global policy count */ @@ -93,6 +105,11 @@ typedef struct { int shutdown_sig; int verbose; int promisc_mode; /**< Promiscuous mode enabled */ + int classifier_enable; + int parse_layer; + int cos_pools; + int pool_size; + int burst_size; } appl_args_t; enum packet_mode { @@ -173,7 +190,7 @@ static inline void print_cls_statistics(appl_args_t *args) printf("\n"); for (i = 0; i < args->policy_count; i++) printf("%-12s |", args->stats[i].cos_name); - printf("Total Packets"); + printf("%-6s %-6s", "Total", "Mpps"); printf("\n"); for (i = 0; i < args->policy_count; i++) printf("%-6s %-6s|", "queue", "pool"); @@ -186,9 +203,12 @@ static inline void print_cls_statistics(appl_args_t *args) if (timeout == 0) infinite = 1; + uint64_t total_packets, last_total_packets = 0; + odp_time_t start = odp_time_local(), end; + float mpps; + for (; timeout > 0 || infinite; timeout--) { sleep(1); - printf("\r"); for (i = 0; i < args->policy_count; i++) { printf("%-6" PRIu64 " ", odp_atomic_load_u64(&args->stats[i] @@ -198,9 +218,13 @@ static inline void print_cls_statistics(appl_args_t *args) .pool_pkt_count)); } - printf("%-" PRIu64, odp_atomic_load_u64(&args-> - total_packets)); - fflush(stdout); + end = odp_time_local(); + total_packets = odp_atomic_load_u64(&args->total_packets); + mpps = (total_packets - last_total_packets) / + (odp_time_diff_ns(end, start) / 1000.0); + printf("%-6" PRIu64 " %-6.3f\n", total_packets, mpps); + last_total_packets = total_packets; + start = end; if (args->shutdown_sig) break; @@ -240,6 +264,9 @@ static odp_pktio_t create_pktio(const char *dev, odp_pool_t pool) odp_pktio_param_t pktio_param; odp_pktin_queue_param_t pktin_param; odp_pktio_capability_t capa; + odp_pktio_config_t cfg; + odp_pktout_queue_param_t pktout_queue_param; + int num_tx; odp_pktio_param_init(&pktio_param); pktio_param.in_mode = ODP_PKTIN_MODE_SCHED; @@ -260,19 +287,36 @@ static odp_pktio_t create_pktio(const char *dev, odp_pool_t pool) } odp_pktin_queue_param_init(&pktin_param); - pktin_param.queue_param.sched.sync = ODP_SCHED_SYNC_ATOMIC; - pktin_param.classifier_enable = 1; + pktin_param.classifier_enable = appl_args_gbl->classifier_enable; if (odp_pktin_queue_config(pktio, &pktin_param)) { ODPH_ERR("pktin queue config failed for %s\n", dev); exit(EXIT_FAILURE); } - if (odp_pktout_queue_config(pktio, NULL)) { + num_tx = appl_args_gbl->cpu_count; + + if (num_tx > (int)capa.max_output_queues) { + printf("Sharing %i output queues between %i workers\n", + capa.max_output_queues, num_tx); + num_tx = capa.max_output_queues; + } + + appl_args_gbl->num_pktout = num_tx; + + odp_pktout_queue_param_init(&pktout_queue_param); + pktout_queue_param.num_queues = num_tx; + + if (odp_pktout_queue_config(pktio, &pktout_queue_param)) { ODPH_ERR("pktout queue config failed for %s\n", dev); exit(EXIT_FAILURE); } + if (odp_pktout_queue(pktio, appl_args_gbl->pktout, num_tx) != num_tx) { + ODPH_ERR("Pktout queue query failed: %s\n", dev); + exit(EXIT_FAILURE); + } + if (appl_args_gbl->promisc_mode && odp_pktio_promisc_mode(pktio) != 1) { if (!capa.set_op.op.promisc_mode) { ODPH_ERR("enabling promisc mode not supported %s\n", dev); @@ -285,11 +329,24 @@ static odp_pktio_t create_pktio(const char *dev, odp_pool_t pool) } } - printf(" created pktio:%02" PRIu64 - ", dev:%s, queue mode (ATOMIC queues)\n" - " \tdefault pktio%02" PRIu64 "\n", - odp_pktio_to_u64(pktio), dev, - odp_pktio_to_u64(pktio)); + printf("created pktio:%" PRIu64 ", dev:%s", odp_pktio_to_u64(pktio), dev); + + odph_ethaddr_t mac; + + if (odp_pktio_mac_addr(pktio, &mac, sizeof(mac)) == sizeof(mac)) { + printf(", mac"); + for (int c = 0; c < (int)sizeof(mac); c++) + printf(":%02x", mac.addr[c]); + } + + printf("\n"); + + odp_pktio_config_init(&cfg); + cfg.parser.layer = appl_args_gbl->parse_layer; + if (odp_pktio_config(pktio, &cfg)) { + ODPH_ERR("failed to configure pktio %s\n", dev); + exit(EXIT_FAILURE); + } return pktio; } @@ -301,89 +358,113 @@ static odp_pktio_t create_pktio(const char *dev, odp_pool_t pool) static int pktio_receive_thread(void *arg) { int thr; - odp_pktout_queue_t pktout; - odp_packet_t pkt; + odp_packet_t pkt[MAX_PKT_BURST]; odp_pool_t pool; - odp_event_t ev; + odp_event_t ev[MAX_PKT_BURST]; odp_queue_t queue; - int i; + int i, j, num, dropped, sent; global_statistics *stats; unsigned long err_cnt = 0; thr = odp_thread_id(); appl_args_t *appl = (appl_args_t *)arg; uint64_t wait_time = odp_schedule_wait_time(100 * ODP_TIME_MSEC_IN_NS); + odp_pktout_queue_t pktout = appl_args_gbl->pktout[thr % appl_args_gbl->num_pktout]; /* Loop packets */ for (;;) { - odp_pktio_t pktio_tmp; - if (appl->shutdown) break; /* Use schedule to get buf from any input queue */ - ev = odp_schedule(&queue, wait_time); + num = odp_schedule_multi(&queue, wait_time, ev, appl_args_gbl->burst_size); /* Loop back to receive packets incase of invalid event */ - if (odp_unlikely(ev == ODP_EVENT_INVALID)) + if (odp_unlikely(!num)) continue; - pkt = odp_packet_from_event(ev); + odp_packet_from_event_multi(pkt, ev, num); - if (appl->verbose) { - odp_queue_info_t info; - uint32_t len = odp_packet_len(pkt); + if (odp_unlikely(appl->verbose)) { + for (j = 0; j < num; j++) { + odp_queue_info_t info; + uint32_t len = odp_packet_len(pkt[j]); - if (odp_queue_info(queue, &info) == 0) - printf("Queue: %s\n", info.name); + if (odp_queue_info(queue, &info) == 0) + printf("Queue: %s\n", info.name); - if (len > 96) - len = 96; + if (len > 96) + len = 96; - odp_packet_print_data(pkt, 0, len); + odp_packet_print_data(pkt[j], 0, len); + } } /* Total packets received */ - odp_atomic_inc_u64(&appl->total_packets); + odp_atomic_add_u64(&appl->total_packets, num); /* Drop packets with errors */ - if (odp_unlikely(drop_err_pkts(&pkt, 1) == 0)) { - ODPH_ERR("Drop frame - err_cnt:%lu\n", ++err_cnt); - continue; + dropped = drop_err_pkts(pkt, num); + if (odp_unlikely(dropped)) { + num -= dropped; + err_cnt += dropped; + ODPH_ERR("Drop frame - err_cnt:%lu\n", err_cnt); } - pktio_tmp = odp_packet_input(pkt); - - if (odp_pktout_queue(pktio_tmp, &pktout, 1) != 1) { - ODPH_ERR(" [%02i] Error: no output queue\n", thr); - return -1; - } + for (j = 0; j < num; j++) { + pool = odp_packet_pool(pkt[j]); - pool = odp_packet_pool(pkt); - - /* Swap Eth MACs and possibly IP-addrs before sending back */ - swap_pkt_addrs(&pkt, 1); - for (i = 0; i < MAX_PMR_COUNT; i++) { - stats = &appl->stats[i]; - if (queue == stats->queue) - odp_atomic_inc_u64(&stats->queue_pkt_count); - if (pool == stats->pool) - odp_atomic_inc_u64(&stats->pool_pkt_count); + for (i = 0; i < MAX_PMR_COUNT; i++) { + stats = &appl->stats[i]; + if (queue == stats->queue) + odp_atomic_inc_u64(&stats->queue_pkt_count); + if (pool == stats->pool) + odp_atomic_inc_u64(&stats->pool_pkt_count); + } } if (appl->appl_mode == APPL_MODE_DROP) { - odp_packet_free(pkt); + odp_packet_free_multi(pkt, num); continue; } - if (odp_pktout_send(pktout, &pkt, 1) < 1) { + /* Swap Eth MACs and possibly IP-addrs before sending back */ + swap_pkt_addrs(pkt, num); + + sent = odp_pktout_send(pktout, pkt, num); + sent = sent < 0 ? 0 : sent; + + if (sent != num) { ODPH_ERR(" [%i] Packet send failed\n", thr); - odp_packet_free(pkt); + odp_packet_free_multi(pkt + sent, num - sent); } } return 0; } +static odp_pool_t pool_create(const char *name) +{ + static odp_pool_t pool = ODP_POOL_INVALID; + odp_pool_param_t pool_params; + + if (!appl_args_gbl->cos_pools && pool != ODP_POOL_INVALID) + return pool; + + odp_pool_param_init(&pool_params); + pool_params.pkt.seg_len = SHM_PKT_POOL_BUF_SIZE; + pool_params.pkt.len = SHM_PKT_POOL_BUF_SIZE; + pool_params.pkt.num = appl_args_gbl->pool_size; + pool_params.type = ODP_POOL_PACKET; + pool = odp_pool_create(name, &pool_params); + + if (pool == ODP_POOL_INVALID) { + ODPH_ERR("Error: failed to create pool %s\n", name); + exit(EXIT_FAILURE); + } + + return pool; +} + static odp_cos_t configure_default_cos(odp_pktio_t pktio, appl_args_t *args) { odp_queue_param_t qparam; @@ -393,14 +474,12 @@ static odp_cos_t configure_default_cos(odp_pktio_t pktio, appl_args_t *args) odp_queue_t queue_default; odp_pool_t pool_default; odp_cos_t cos_default; - odp_pool_param_t pool_params; odp_cls_cos_param_t cls_param; global_statistics *stats = args->stats; odp_queue_param_init(&qparam); qparam.type = ODP_QUEUE_TYPE_SCHED; - qparam.sched.prio = odp_schedule_default_prio(); qparam.sched.sync = ODP_SCHED_SYNC_PARALLEL; qparam.sched.group = ODP_SCHED_GROUP_ALL; queue_default = odp_queue_create(queue_name, &qparam); @@ -409,17 +488,7 @@ static odp_cos_t configure_default_cos(odp_pktio_t pktio, appl_args_t *args) exit(EXIT_FAILURE); } - odp_pool_param_init(&pool_params); - pool_params.pkt.seg_len = SHM_PKT_POOL_BUF_SIZE; - pool_params.pkt.len = SHM_PKT_POOL_BUF_SIZE; - pool_params.pkt.num = SHM_PKT_POOL_SIZE / SHM_PKT_POOL_BUF_SIZE; - pool_params.type = ODP_POOL_PACKET; - pool_default = odp_pool_create(pool_name, &pool_params); - - if (pool_default == ODP_POOL_INVALID) { - ODPH_ERR("Error: default pool create failed\n"); - exit(EXIT_FAILURE); - } + pool_default = pool_create(pool_name); odp_cls_cos_param_init(&cls_param); cls_param.pool = pool_default; @@ -439,7 +508,8 @@ static odp_cos_t configure_default_cos(odp_pktio_t pktio, appl_args_t *args) stats[args->policy_count].cos = cos_default; /* add default queue to global stats */ stats[args->policy_count].queue = queue_default; - stats[args->policy_count].pool = pool_default; + if (appl_args_gbl->cos_pools) + stats[args->policy_count].pool = pool_default; snprintf(stats[args->policy_count].cos_name, sizeof(stats[args->policy_count].cos_name), "%s", cos_name); @@ -471,7 +541,6 @@ static void configure_cos(odp_cos_t default_cos, appl_args_t *args) char cos_name[ODP_COS_NAME_LEN]; char pool_name[ODP_POOL_NAME_LEN]; const char *queue_name; - odp_pool_param_t pool_params; odp_cls_cos_param_t cls_param; int i; global_statistics *stats; @@ -482,7 +551,6 @@ static void configure_cos(odp_cos_t default_cos, appl_args_t *args) odp_queue_param_init(&qparam); qparam.type = ODP_QUEUE_TYPE_SCHED; - qparam.sched.prio = i % odp_schedule_num_prio(); qparam.sched.sync = ODP_SCHED_SYNC_PARALLEL; qparam.sched.group = ODP_SCHED_GROUP_ALL; @@ -493,26 +561,15 @@ static void configure_cos(odp_cos_t default_cos, appl_args_t *args) exit(EXIT_FAILURE); } - odp_pool_param_init(&pool_params); - pool_params.pkt.seg_len = SHM_PKT_POOL_BUF_SIZE; - pool_params.pkt.len = SHM_PKT_POOL_BUF_SIZE; - pool_params.pkt.num = SHM_PKT_POOL_SIZE / - SHM_PKT_POOL_BUF_SIZE; - pool_params.type = ODP_POOL_PACKET; - snprintf(pool_name, sizeof(pool_name), "%sPool%d", args->stats[i].cos_name, i); - stats->pool = odp_pool_create(pool_name, &pool_params); - - if (stats->pool == ODP_POOL_INVALID) { - ODPH_ERR("Error: default pool create failed\n"); - exit(EXIT_FAILURE); - } snprintf(cos_name, sizeof(cos_name), "CoS%s", stats->cos_name); odp_cls_cos_param_init(&cls_param); - cls_param.pool = stats->pool; + cls_param.pool = pool_create(pool_name); + if (appl_args_gbl->cos_pools) + stats->pool = cls_param.pool; cls_param.queue = stats->queue; cls_param.drop_policy = ODP_COS_DROP_POOL; stats->cos = odp_cls_cos_create(cos_name, &cls_param); @@ -572,7 +629,6 @@ int main(int argc, char *argv[]) int i; odp_cpumask_t cpumask; char cpumaskstr[ODP_CPUMASK_STR_SIZE]; - odp_pool_param_t params; odp_pktio_t pktio; appl_args_t *args; odp_cos_t default_cos; @@ -645,18 +701,7 @@ int main(int argc, char *argv[]) printf("cpu mask: %s\n", cpumaskstr); /* Create packet pool */ - odp_pool_param_init(¶ms); - params.pkt.seg_len = SHM_PKT_POOL_BUF_SIZE; - params.pkt.len = SHM_PKT_POOL_BUF_SIZE; - params.pkt.num = SHM_PKT_POOL_SIZE / SHM_PKT_POOL_BUF_SIZE; - params.type = ODP_POOL_PACKET; - - pool = odp_pool_create("packet_pool", ¶ms); - - if (pool == ODP_POOL_INVALID) { - ODPH_ERR("Error: packet pool create failed\n"); - exit(EXIT_FAILURE); - } + pool = pool_create("packet_pool"); /* Configure scheduler */ odp_schedule_config(NULL); @@ -673,6 +718,7 @@ int main(int argc, char *argv[]) configure_cos(default_cos, args); printf("\n"); + odp_pool_print_all(); odp_cls_print_all(); if (odp_pktio_start(pktio)) { @@ -725,7 +771,7 @@ int main(int argc, char *argv[]) ODPH_ERR("err: odp_cos_destroy for %d\n", i); if (odp_queue_destroy(args->stats[i].queue)) ODPH_ERR("err: odp_queue_destroy for %d\n", i); - if (odp_pool_destroy(args->stats[i].pool)) + if (args->cos_pools && odp_pool_destroy(args->stats[i].pool)) ODPH_ERR("err: odp_pool_destroy for %d\n", i); } @@ -756,26 +802,26 @@ args_error: * @param pkt_tbl Array of packet * @param len Length of pkt_tbl[] * - * @return Number of packets with no detected error + * @return Number of packets dropped */ static int drop_err_pkts(odp_packet_t pkt_tbl[], unsigned len) { odp_packet_t pkt; - unsigned pkt_cnt = len; unsigned i, j; + int dropped = 0; for (i = 0, j = 0; i < len; ++i) { pkt = pkt_tbl[i]; if (odp_unlikely(odp_packet_has_error(pkt))) { odp_packet_free(pkt); /* Drop */ - pkt_cnt--; + dropped++; } else if (odp_unlikely(i != j++)) { pkt_tbl[j-1] = pkt; } } - return pkt_cnt; + return dropped; } /** @@ -1114,7 +1160,6 @@ static int parse_args(int argc, char *argv[], appl_args_t *appl_args) size_t len; int i; int interface = 0; - int policy = 0; int ret = 0; static const struct option longopts[] = { @@ -1127,14 +1172,24 @@ static int parse_args(int argc, char *argv[], appl_args_t *appl_args) {"promisc_mode", no_argument, NULL, 'P'}, {"verbose", no_argument, NULL, 'v'}, {"help", no_argument, NULL, 'h'}, + {"enable", required_argument, NULL, 'e'}, + {"layer", required_argument, NULL, 'l'}, + {"dedicated", required_argument, NULL, 'd'}, + {"size", required_argument, NULL, 's'}, + {"burst", required_argument, NULL, 'b'}, {NULL, 0, NULL, 0} }; - static const char *shortopts = "+c:t:i:p:m:t:C:Pvh"; + static const char *shortopts = "+c:t:i:p:m:t:C:Pvhe:l:d:s:b:"; appl_args->cpu_count = 1; /* Use one worker by default */ appl_args->verbose = 0; appl_args->promisc_mode = 0; + appl_args->classifier_enable = 1; + appl_args->parse_layer = ODP_PROTO_LAYER_ALL; + appl_args->cos_pools = 1; + appl_args->pool_size = SHM_PKT_POOL_SIZE; + appl_args->burst_size = DEF_PKT_BURST; while (ret == 0) { opt = getopt_long(argc, argv, shortopts, @@ -1152,7 +1207,6 @@ static int parse_args(int argc, char *argv[], appl_args_t *appl_args) ret = -1; break; } - policy = 1; break; case 't': appl_args->time = atoi(optarg); @@ -1196,12 +1250,27 @@ static int parse_args(int argc, char *argv[], appl_args_t *appl_args) case 'h': ret = -1; break; + case 'e': + appl_args->classifier_enable = atoi(optarg); + break; + case 'l': + appl_args->parse_layer = atoi(optarg); + break; + case 'd': + appl_args->cos_pools = atoi(optarg); + break; + case 's': + appl_args->pool_size = atoi(optarg); + break; + case 'b': + appl_args->burst_size = atoi(optarg); + break; default: break; } } - if (!interface || !policy) + if (!interface) ret = -1; if (appl_args->if_name == NULL) @@ -1254,6 +1323,8 @@ static void usage(void) "\n" "Mandatory OPTIONS:\n" " -i, --interface <interface name>\n" + "\n" + "Optional OPTIONS\n" " -p, --policy <PMR term>:<offset>:<value>:<mask>:<src queue>:<dst queue>\n" "\n" " <PMR term> PMR term name defined in odp_cls_pmr_term_t\n" @@ -1266,17 +1337,33 @@ static void usage(void) " this is not defined.\n" " <dst queue> Name of the destination queue (CoS).\n" "\n" - "Optional OPTIONS\n" " -c, --count <num> CPU count, 0=all available, default=1\n" "\n" " -m, --mode <mode> 0: Packet Drop mode. Received packets will be dropped\n" - " !0: Packet ICMP mode. Received packets will be sent back\n" + " !0: Echo mode. Received packets will be sent back\n" " default: Packet Drop mode\n" "\n" " -t, --time <sec> !0: Time for which the classifier will be run in seconds\n" " 0: Runs in infinite loop\n" " default: Runs in infinite loop\n" "\n" + " -e, --enable <enable> 0: Classifier is disabled\n" + " 1: Classifier is enabled\n" + " default: Classifier is enabled\n" + "\n" + " -l, --layer <layer> Parse packets up to and including this layer. See odp_proto_layer_t\n" + " default: ODP_PROTO_LAYER_ALL\n" + "\n" + " -d, --dedicated <enable> 0: One pool for pktio and all CoSes\n" + " 1: Dedicated pools for pktio and each CoS\n" + " default: Dedicated pools\n" + "\n" + " -s, --size <num> Number of packets in each packet pool\n" + " default: %d\n" + "\n" + " -b, --burst <num> Packet burst size\n" + " default: %d\n" + "\n" " -C, --ci_pass <dst queue:count>\n" " Minimum acceptable packet count for a CoS destination queue.\n" " If the received packet count is smaller than this value,\n" @@ -1285,5 +1372,5 @@ static void usage(void) " -P, --promisc_mode Enable promiscuous mode.\n" " -v, --verbose Verbose output.\n" " -h, --help Display help and exit.\n" - "\n"); + "\n", SHM_PKT_POOL_SIZE, DEF_PKT_BURST); } diff --git a/example/sysinfo/odp_sysinfo.c b/example/sysinfo/odp_sysinfo.c index 4df26dbb2..523f70496 100644 --- a/example/sysinfo/odp_sysinfo.c +++ b/example/sysinfo/odp_sysinfo.c @@ -177,53 +177,93 @@ static const char *auth_alg_name(odp_auth_alg_t auth) } } -static void print_cipher_algos(odp_crypto_cipher_algos_t ciphers) +typedef void (*cipher_op_t)(odp_cipher_alg_t alg); +typedef void (*auth_op_t)(odp_auth_alg_t alg); + +static void foreach_cipher(odp_crypto_cipher_algos_t ciphers, cipher_op_t op) { if (ciphers.bit.null) - printf("%s ", cipher_alg_name(ODP_CIPHER_ALG_NULL)); + op(ODP_CIPHER_ALG_NULL); if (ciphers.bit.des) - printf("%s ", cipher_alg_name(ODP_CIPHER_ALG_DES)); + op(ODP_CIPHER_ALG_DES); if (ciphers.bit.trides_cbc) - printf("%s ", cipher_alg_name(ODP_CIPHER_ALG_3DES_CBC)); + op(ODP_CIPHER_ALG_3DES_CBC); + if (ciphers.bit.trides_ecb) + op(ODP_CIPHER_ALG_3DES_ECB); if (ciphers.bit.aes_cbc) - printf("%s ", cipher_alg_name(ODP_CIPHER_ALG_AES_CBC)); + op(ODP_CIPHER_ALG_AES_CBC); if (ciphers.bit.aes_ctr) - printf("%s ", cipher_alg_name(ODP_CIPHER_ALG_AES_CTR)); + op(ODP_CIPHER_ALG_AES_CTR); + if (ciphers.bit.aes_ecb) + op(ODP_CIPHER_ALG_AES_ECB); + if (ciphers.bit.aes_cfb128) + op(ODP_CIPHER_ALG_AES_CFB128); + if (ciphers.bit.aes_xts) + op(ODP_CIPHER_ALG_AES_XTS); if (ciphers.bit.aes_gcm) - printf("%s ", cipher_alg_name(ODP_CIPHER_ALG_AES_GCM)); + op(ODP_CIPHER_ALG_AES_GCM); if (ciphers.bit.aes_ccm) - printf("%s ", cipher_alg_name(ODP_CIPHER_ALG_AES_CCM)); + op(ODP_CIPHER_ALG_AES_CCM); if (ciphers.bit.chacha20_poly1305) - printf("%s ", - cipher_alg_name(ODP_CIPHER_ALG_CHACHA20_POLY1305)); + op(ODP_CIPHER_ALG_CHACHA20_POLY1305); + if (ciphers.bit.kasumi_f8) + op(ODP_CIPHER_ALG_KASUMI_F8); + if (ciphers.bit.snow3g_uea2) + op(ODP_CIPHER_ALG_SNOW3G_UEA2); + if (ciphers.bit.aes_eea2) + op(ODP_CIPHER_ALG_AES_EEA2); + if (ciphers.bit.zuc_eea3) + op(ODP_CIPHER_ALG_ZUC_EEA3); } -static void print_auth_algos(odp_crypto_auth_algos_t auths) +static void foreach_auth(odp_crypto_auth_algos_t auths, auth_op_t op) { if (auths.bit.null) - printf("%s ", auth_alg_name(ODP_AUTH_ALG_NULL)); + op(ODP_AUTH_ALG_NULL); if (auths.bit.md5_hmac) - printf("%s ", auth_alg_name(ODP_AUTH_ALG_MD5_HMAC)); + op(ODP_AUTH_ALG_MD5_HMAC); if (auths.bit.sha1_hmac) - printf("%s ", auth_alg_name(ODP_AUTH_ALG_SHA1_HMAC)); + op(ODP_AUTH_ALG_SHA1_HMAC); + if (auths.bit.sha224_hmac) + op(ODP_AUTH_ALG_SHA224_HMAC); if (auths.bit.sha256_hmac) - printf("%s ", auth_alg_name(ODP_AUTH_ALG_SHA256_HMAC)); + op(ODP_AUTH_ALG_SHA256_HMAC); if (auths.bit.sha384_hmac) - printf("%s ", auth_alg_name(ODP_AUTH_ALG_SHA384_HMAC)); + op(ODP_AUTH_ALG_SHA384_HMAC); if (auths.bit.sha512_hmac) - printf("%s ", auth_alg_name(ODP_AUTH_ALG_SHA512_HMAC)); + op(ODP_AUTH_ALG_SHA512_HMAC); if (auths.bit.aes_gcm) - printf("%s ", auth_alg_name(ODP_AUTH_ALG_AES_GCM)); + op(ODP_AUTH_ALG_AES_GCM); if (auths.bit.aes_gmac) - printf("%s ", auth_alg_name(ODP_AUTH_ALG_AES_GMAC)); + op(ODP_AUTH_ALG_AES_GMAC); if (auths.bit.aes_ccm) - printf("%s ", auth_alg_name(ODP_AUTH_ALG_AES_CCM)); + op(ODP_AUTH_ALG_AES_CCM); if (auths.bit.aes_cmac) - printf("%s ", auth_alg_name(ODP_AUTH_ALG_AES_CMAC)); + op(ODP_AUTH_ALG_AES_CMAC); if (auths.bit.aes_xcbc_mac) - printf("%s ", auth_alg_name(ODP_AUTH_ALG_AES_XCBC_MAC)); + op(ODP_AUTH_ALG_AES_XCBC_MAC); if (auths.bit.chacha20_poly1305) - printf("%s ", auth_alg_name(ODP_AUTH_ALG_CHACHA20_POLY1305)); + op(ODP_AUTH_ALG_CHACHA20_POLY1305); + if (auths.bit.kasumi_f9) + op(ODP_AUTH_ALG_KASUMI_F9); + if (auths.bit.snow3g_uia2) + op(ODP_AUTH_ALG_SNOW3G_UIA2); + if (auths.bit.aes_eia2) + op(ODP_AUTH_ALG_AES_EIA2); + if (auths.bit.zuc_eia3) + op(ODP_AUTH_ALG_ZUC_EIA3); + if (auths.bit.md5) + op(ODP_AUTH_ALG_MD5); + if (auths.bit.sha1) + op(ODP_AUTH_ALG_SHA1); + if (auths.bit.sha224) + op(ODP_AUTH_ALG_SHA224); + if (auths.bit.sha256) + op(ODP_AUTH_ALG_SHA256); + if (auths.bit.sha384) + op(ODP_AUTH_ALG_SHA384); + if (auths.bit.sha512) + op(ODP_AUTH_ALG_SHA512); } static void print_cipher_capa(odp_cipher_alg_t cipher) @@ -275,52 +315,14 @@ static void print_auth_capa(odp_auth_alg_t auth) } } -static void print_cipher_caps(odp_crypto_cipher_algos_t ciphers) +static void print_cipher(odp_cipher_alg_t alg) { - if (ciphers.bit.null) - print_cipher_capa(ODP_CIPHER_ALG_NULL); - if (ciphers.bit.des) - print_cipher_capa(ODP_CIPHER_ALG_DES); - if (ciphers.bit.trides_cbc) - print_cipher_capa(ODP_CIPHER_ALG_3DES_CBC); - if (ciphers.bit.aes_cbc) - print_cipher_capa(ODP_CIPHER_ALG_AES_CBC); - if (ciphers.bit.aes_ctr) - print_cipher_capa(ODP_CIPHER_ALG_AES_CTR); - if (ciphers.bit.aes_gcm) - print_cipher_capa(ODP_CIPHER_ALG_AES_GCM); - if (ciphers.bit.aes_ccm) - print_cipher_capa(ODP_CIPHER_ALG_AES_CCM); - if (ciphers.bit.chacha20_poly1305) - print_cipher_capa(ODP_CIPHER_ALG_CHACHA20_POLY1305); + printf("%s ", cipher_alg_name(alg)); } -static void print_auth_caps(odp_crypto_auth_algos_t auths) +static void print_auth(odp_auth_alg_t alg) { - if (auths.bit.null) - print_auth_capa(ODP_AUTH_ALG_NULL); - if (auths.bit.md5_hmac) - print_auth_capa(ODP_AUTH_ALG_MD5_HMAC); - if (auths.bit.sha1_hmac) - print_auth_capa(ODP_AUTH_ALG_SHA1_HMAC); - if (auths.bit.sha256_hmac) - print_auth_capa(ODP_AUTH_ALG_SHA256_HMAC); - if (auths.bit.sha384_hmac) - print_auth_capa(ODP_AUTH_ALG_SHA384_HMAC); - if (auths.bit.sha512_hmac) - print_auth_capa(ODP_AUTH_ALG_SHA512_HMAC); - if (auths.bit.aes_gcm) - print_auth_capa(ODP_AUTH_ALG_AES_GCM); - if (auths.bit.aes_gmac) - print_auth_capa(ODP_AUTH_ALG_AES_GMAC); - if (auths.bit.aes_ccm) - print_auth_capa(ODP_AUTH_ALG_AES_CCM); - if (auths.bit.aes_cmac) - print_auth_capa(ODP_AUTH_ALG_AES_CMAC); - if (auths.bit.aes_xcbc_mac) - print_auth_capa(ODP_AUTH_ALG_AES_XCBC_MAC); - if (auths.bit.chacha20_poly1305) - print_auth_capa(ODP_AUTH_ALG_CHACHA20_POLY1305); + printf("%s ", auth_alg_name(alg)); } int main(void) @@ -626,13 +628,13 @@ int main(void) printf(" queue_type_sched: %i\n", crypto_capa.queue_type_sched); printf(" queue_type_plain: %i\n", crypto_capa.queue_type_plain); printf(" cipher algorithms: "); - print_cipher_algos(crypto_capa.ciphers); + foreach_cipher(crypto_capa.ciphers, print_cipher); printf("\n"); - print_cipher_caps(crypto_capa.ciphers); + foreach_cipher(crypto_capa.ciphers, print_cipher_capa); printf(" auth algorithms: "); - print_auth_algos(crypto_capa.auths); + foreach_auth(crypto_capa.auths, print_auth); printf("\n"); - print_auth_caps(crypto_capa.auths); + foreach_auth(crypto_capa.auths, print_auth_capa); printf("\n"); } @@ -675,10 +677,10 @@ int main(void) printf(" inline TM pipelining: %s\n", support_level(ipsec_capa.inline_ipsec_tm)); printf(" cipher algorithms: "); - print_cipher_algos(ipsec_capa.ciphers); + foreach_cipher(ipsec_capa.ciphers, print_cipher); printf("\n"); printf(" auth algorithms: "); - print_auth_algos(ipsec_capa.auths); + foreach_auth(ipsec_capa.auths, print_auth); printf("\n\n"); } diff --git a/helper/threads.c b/helper/threads.c index 1b5df8965..74f64b138 100644 --- a/helper/threads.c +++ b/helper/threads.c @@ -20,6 +20,7 @@ #include <sys/mman.h> #include <sys/time.h> #include <sys/resource.h> +#include <inttypes.h> #include <odp_api.h> #include <odp/helper/threads.h> @@ -361,7 +362,10 @@ int odph_thread_create(odph_thread_t thread[], } while (status != INIT_DONE && timeout == 0); if (timeout) { - ODPH_ERR("Thread (i:%i) start up timeout\n", i); + ODPH_ERR("Thread (i:%i) start up timeout: sync timeout %" PRIu64 "" + " , t1 %" PRIu64 ", t2 %" PRIu64 "\n", i, + param->sync_timeout, odp_time_to_ns(t1), + odp_time_to_ns(t2)); break; } } diff --git a/include/odp/api/abi-default/schedule.h b/include/odp/api/abi-default/schedule.h index f43ea7802..89b92297b 100644 --- a/include/odp/api/abi-default/schedule.h +++ b/include/odp/api/abi-default/schedule.h @@ -4,12 +4,6 @@ * SPDX-License-Identifier: BSD-3-Clause */ -/** - * @file - * - * ODP schedule - */ - #ifndef ODP_ABI_SCHEDULE_H_ #define ODP_ABI_SCHEDULE_H_ @@ -17,20 +11,7 @@ extern "C" { #endif -#include <odp/api/std_types.h> - -/** @addtogroup odp_scheduler - * @{ - */ - -#define ODP_SCHED_WAIT UINT64_MAX -#define ODP_SCHED_NO_WAIT 0 - -#define ODP_SCHED_GROUP_NAME_LEN 32 - -/** - * @} - */ +/* Empty header required due to the schedule inline functions */ #ifdef __cplusplus } diff --git a/include/odp/api/abi-default/schedule_types.h b/include/odp/api/abi-default/schedule_types.h index b386b6b4d..5e91c6e61 100644 --- a/include/odp/api/abi-default/schedule_types.h +++ b/include/odp/api/abi-default/schedule_types.h @@ -18,11 +18,17 @@ extern "C" { #endif #include <odp/api/deprecated.h> +#include <odp/api/std_types.h> /** @addtogroup odp_scheduler * @{ */ +#define ODP_SCHED_WAIT UINT64_MAX +#define ODP_SCHED_NO_WAIT 0 + +#define ODP_SCHED_GROUP_NAME_LEN 32 + #if ODP_DEPRECATED_API #define ODP_SCHED_PRIO_HIGHEST (odp_schedule_max_prio()) diff --git a/include/odp/api/schedule.h b/include/odp/api/schedule.h index 88d1712b5..4d97634b8 100644 --- a/include/odp/api/schedule.h +++ b/include/odp/api/schedule.h @@ -17,7 +17,6 @@ extern "C" { #endif -#include <odp/api/abi/schedule_types.h> #include <odp/api/abi/schedule.h> #include <odp/api/spec/schedule.h> diff --git a/include/odp/api/spec/schedule.h b/include/odp/api/spec/schedule.h index 29db262c0..417e5cb22 100644 --- a/include/odp/api/spec/schedule.h +++ b/include/odp/api/spec/schedule.h @@ -30,21 +30,6 @@ extern "C" { */ /** - * @def ODP_SCHED_WAIT - * Wait infinitely - */ - -/** - * @def ODP_SCHED_NO_WAIT - * Do not wait - */ - -/** - * @def ODP_SCHED_GROUP_NAME_LEN - * Maximum schedule group name length in chars including null char - */ - -/** * Schedule wait time * * Converts nanoseconds to wait values for other schedule functions. @@ -427,14 +412,6 @@ int odp_schedule_group_thrmask(odp_schedule_group_t group, odp_thrmask_t *thrmask); /** - * Schedule group information - */ -typedef struct odp_schedule_group_info_t { - const char *name; /**< Schedule group name */ - odp_thrmask_t thrmask; /**< Thread mask of the schedule group */ -} odp_schedule_group_info_t; - -/** * Retrieve information about a schedule group * * Fills in schedule group information structure with current values. diff --git a/include/odp/api/spec/schedule_types.h b/include/odp/api/spec/schedule_types.h index b90cd5062..aafc52513 100644 --- a/include/odp/api/spec/schedule_types.h +++ b/include/odp/api/spec/schedule_types.h @@ -15,6 +15,7 @@ #include <odp/visibility_begin.h> #include <odp/api/std_types.h> +#include <odp/api/thrmask.h> #ifdef __cplusplus extern "C" { @@ -25,6 +26,21 @@ extern "C" { */ /** + * @def ODP_SCHED_WAIT + * Wait infinitely + */ + +/** + * @def ODP_SCHED_NO_WAIT + * Do not wait + */ + +/** + * @def ODP_SCHED_GROUP_NAME_LEN + * Maximum schedule group name length in chars including null char + */ + +/** * @def ODP_SCHED_PRIO_HIGHEST * @deprecated This macro is equivalent of calling odp_schedule_max_prio(). Use * direct function call instead. @@ -295,6 +311,14 @@ typedef struct odp_schedule_config_t { } odp_schedule_config_t; /** + * Schedule group information + */ +typedef struct odp_schedule_group_info_t { + const char *name; /**< Schedule group name */ + odp_thrmask_t thrmask; /**< Thread mask of the schedule group */ +} odp_schedule_group_info_t; + +/** * @} */ diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index a676380dd..9d24d2e6c 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -29,8 +29,10 @@ odpapiplatincludedir= $(includedir)/odp/api/plat odpapiplatinclude_HEADERS = \ include/odp/api/plat/atomic_inlines.h \ include/odp/api/plat/buffer_inlines.h \ + include/odp/api/plat/buffer_inline_types.h \ include/odp/api/plat/byteorder_inlines.h \ include/odp/api/plat/cpu_inlines.h \ + include/odp/api/plat/debug_inlines.h \ include/odp/api/plat/event_inlines.h \ include/odp/api/plat/event_inline_types.h \ include/odp/api/plat/event_vector_inline_types.h \ @@ -44,12 +46,17 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/pool_inline_types.h \ include/odp/api/plat/queue_inlines.h \ include/odp/api/plat/queue_inline_types.h \ + include/odp/api/plat/schedule_inlines.h \ + include/odp/api/plat/schedule_inline_types.h \ + include/odp/api/plat/rwlock_inlines.h \ + include/odp/api/plat/rwlock_recursive_inlines.h \ include/odp/api/plat/spinlock_inlines.h \ include/odp/api/plat/spinlock_recursive_inlines.h \ include/odp/api/plat/std_inlines.h \ include/odp/api/plat/strong_types.h \ include/odp/api/plat/sync_inlines.h \ include/odp/api/plat/thread_inlines.h \ + include/odp/api/plat/thread_inline_types.h \ include/odp/api/plat/ticketlock_inlines.h \ include/odp/api/plat/time_inlines.h \ include/odp/api/plat/timer_inlines.h \ @@ -215,8 +222,6 @@ __LIB__libodp_dpdk_la_SOURCES = \ ../linux-generic/odp_random.c \ ../linux-generic/odp_random_std.c \ ../linux-generic/odp_random_openssl.c \ - ../linux-generic/odp_rwlock.c \ - ../linux-generic/odp_rwlock_recursive.c \ ../linux-generic/odp_schedule_basic.c \ odp_schedule_eventdev.c \ odp_schedule_if.c \ @@ -249,6 +254,9 @@ __LIB__libodp_dpdk_la_SOURCES += \ ../linux-generic/odp_packet_io_api.c \ ../linux-generic/odp_pool_api.c \ ../linux-generic/odp_queue_api.c \ + ../linux-generic/odp_schedule_api.c \ + ../linux-generic/odp_rwlock_api.c \ + ../linux-generic/odp_rwlock_recursive_api.c \ ../linux-generic/odp_spinlock_api.c \ ../linux-generic/odp_spinlock_recursive_api.c \ odp_std_api.c \ diff --git a/platform/linux-dpdk/include/odp/api/plat/buffer_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/buffer_inline_types.h new file mode 120000 index 000000000..4ef8ae6f2 --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/buffer_inline_types.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/buffer_inline_types.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h index a47f9ad70..3aa399399 100644 --- a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h @@ -7,12 +7,14 @@ #ifndef ODP_PLAT_BUFFER_INLINES_H_ #define ODP_PLAT_BUFFER_INLINES_H_ -#include <odp/api/event_types.h> +#include <odp/api/event.h> #include <odp/api/hints.h> #include <odp/api/pool_types.h> #include <odp/api/abi/buffer.h> +#include <odp/api/plat/buffer_inline_types.h> +#include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/event_inline_types.h> #include <rte_mbuf.h> @@ -28,6 +30,7 @@ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ extern const _odp_event_inline_offset_t _odp_event_inline_offset; +extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; #ifndef _ODP_NO_INLINE /* Inline functions by default */ @@ -37,14 +40,18 @@ extern const _odp_event_inline_offset_t _odp_event_inline_offset; #define odp_buffer_addr __odp_buffer_addr #define odp_buffer_size __odp_buffer_size #define odp_buffer_pool __odp_buffer_pool + #define odp_buffer_user_area __odp_buffer_user_area #define odp_buffer_free __odp_buffer_free #define odp_buffer_free_multi __odp_buffer_free_multi + #define odp_buffer_is_valid __odp_buffer_is_valid #else #define _ODP_INLINE #endif _ODP_INLINE odp_buffer_t odp_buffer_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_BUFFER); + return (odp_buffer_t)ev; } @@ -68,6 +75,11 @@ _ODP_INLINE odp_pool_t odp_buffer_pool(odp_buffer_t buf) return (odp_pool_t)(uintptr_t)_odp_event_hdr_field(buf, void *, pool); } +_ODP_INLINE void *odp_buffer_user_area(odp_buffer_t buf) +{ + return _odp_buffer_get(buf, void *, uarea_addr); +} + _ODP_INLINE void odp_buffer_free(odp_buffer_t buf) { struct rte_mbuf *mbuf = (struct rte_mbuf *)buf; @@ -103,6 +115,17 @@ _ODP_INLINE void odp_buffer_free_multi(const odp_buffer_t buf[], int num) rte_mempool_put_bulk(mp_pending, (void **)mbuf_tbl, num_pending); } +_ODP_INLINE int odp_buffer_is_valid(odp_buffer_t buf) +{ + if (odp_event_is_valid(odp_buffer_to_event(buf)) == 0) + return 0; + + if (odp_event_type(odp_buffer_to_event(buf)) != ODP_EVENT_BUFFER) + return 0; + + return 1; +} + /** @endcond */ #endif diff --git a/platform/linux-dpdk/include/odp/api/plat/debug_inlines.h b/platform/linux-dpdk/include/odp/api/plat/debug_inlines.h new file mode 120000 index 000000000..05e4e91d3 --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/debug_inlines.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/debug_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h index 26ada3655..4bf00dac4 100644 --- a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h @@ -42,6 +42,7 @@ typedef struct _odp_packet_inline_offset_t { uint16_t input_flags; uint16_t flags; uint16_t subtype; + uint16_t cls_mark; uint16_t buf_addr; uint16_t data; uint16_t pkt_len; diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h index 9856d1dd7..76559ba86 100644 --- a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h @@ -18,16 +18,13 @@ extern "C" { #endif +#include <odp/api/event.h> #include <odp/api/hints.h> #include <odp/api/packet_types.h> #include <odp/api/pool_types.h> #include <odp/api/time.h> -#include <odp/api/abi/buffer.h> -#include <odp/api/abi/event_types.h> -#include <odp/api/abi/packet.h> -#include <odp/api/abi/packet_io.h> - +#include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/packet_io_inlines.h> #include <odp/api/plat/packet_inline_types.h> #include <odp/api/plat/pool_inline_types.h> @@ -70,6 +67,7 @@ extern "C" { #define odp_packet_input_index __odp_packet_input_index #define odp_packet_num_segs __odp_packet_num_segs #define odp_packet_user_ptr __odp_packet_user_ptr + #define odp_packet_user_ptr_set __odp_packet_user_ptr_set #define odp_packet_user_area __odp_packet_user_area #define odp_packet_user_area_size __odp_packet_user_area_size #define odp_packet_user_flag __odp_packet_user_flag @@ -77,12 +75,24 @@ extern "C" { #define odp_packet_l2_offset __odp_packet_l2_offset #define odp_packet_l3_offset __odp_packet_l3_offset #define odp_packet_l4_offset __odp_packet_l4_offset + #define odp_packet_l2_offset_set __odp_packet_l2_offset_set + #define odp_packet_l3_offset_set __odp_packet_l3_offset_set + #define odp_packet_l4_offset_set __odp_packet_l4_offset_set #define odp_packet_l2_ptr __odp_packet_l2_ptr #define odp_packet_l3_ptr __odp_packet_l3_ptr #define odp_packet_l4_ptr __odp_packet_l4_ptr + #define odp_packet_l2_type __odp_packet_l2_type + #define odp_packet_l3_type __odp_packet_l3_type + #define odp_packet_l4_type __odp_packet_l4_type + #define odp_packet_l3_chksum_status __odp_packet_l3_chksum_status + #define odp_packet_l4_chksum_status __odp_packet_l4_chksum_status + #define odp_packet_l3_chksum_insert __odp_packet_l3_chksum_insert + #define odp_packet_l4_chksum_insert __odp_packet_l4_chksum_insert #define odp_packet_flow_hash __odp_packet_flow_hash #define odp_packet_flow_hash_set __odp_packet_flow_hash_set #define odp_packet_ts __odp_packet_ts + #define odp_packet_ts_set __odp_packet_ts_set + #define odp_packet_ts_request __odp_packet_ts_request #define odp_packet_head __odp_packet_head #define odp_packet_is_segmented __odp_packet_is_segmented #define odp_packet_first_seg __odp_packet_first_seg @@ -108,6 +118,7 @@ extern "C" { #define odp_packet_color __odp_packet_color #define odp_packet_drop_eligible __odp_packet_drop_eligible #define odp_packet_shaper_len_adjust __odp_packet_shaper_len_adjust + #define odp_packet_cls_mark __odp_packet_cls_mark #define odp_packet_buf_data_len __odp_packet_buf_data_len #define odp_packet_buf_size __odp_packet_buf_size #define odp_packet_buf_head __odp_packet_buf_head @@ -238,6 +249,20 @@ _ODP_INLINE void *odp_packet_user_ptr(odp_packet_t pkt) return _odp_pkt_get(pkt, void *, user_ptr); } +_ODP_INLINE void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ptr) +{ + _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + const void **user_ptr = _odp_pkt_get_ptr(pkt, const void *, user_ptr); + + if (odp_unlikely(ptr == NULL)) { + flags->user_ptr_set = 0; + return; + } + + *user_ptr = ptr; + flags->user_ptr_set = 1; +} + _ODP_INLINE void *odp_packet_user_area(odp_packet_t pkt) { return _odp_pkt_get(pkt, void *, user_area); @@ -281,6 +306,42 @@ _ODP_INLINE uint32_t odp_packet_l4_offset(odp_packet_t pkt) return _odp_pkt_get(pkt, uint16_t, l4_offset); } +_ODP_INLINE int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset) +{ + uint16_t *l2_offset = _odp_pkt_get_ptr(pkt, uint16_t, l2_offset); + _odp_packet_input_flags_t *input_flags = _odp_pkt_get_ptr(pkt, _odp_packet_input_flags_t, + input_flags); + + if (odp_unlikely(offset >= odp_packet_len(pkt))) + return -1; + + input_flags->l2 = 1; + *l2_offset = (uint16_t)offset; + return 0; +} + +_ODP_INLINE int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset) +{ + uint16_t *l3_offset = _odp_pkt_get_ptr(pkt, uint16_t, l3_offset); + + if (odp_unlikely(offset >= odp_packet_len(pkt))) + return -1; + + *l3_offset = (uint16_t)offset; + return 0; +} + +_ODP_INLINE int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset) +{ + uint16_t *l4_offset = _odp_pkt_get_ptr(pkt, uint16_t, l4_offset); + + if (odp_unlikely(offset >= odp_packet_len(pkt))) + return -1; + + *l4_offset = (uint16_t)offset; + return 0; +} + _ODP_INLINE void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len) { return odp_packet_offset(pkt, _odp_pkt_get(pkt, uint16_t, l2_offset), @@ -299,6 +360,107 @@ _ODP_INLINE void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len) len, NULL); } +_ODP_INLINE odp_proto_l2_type_t odp_packet_l2_type(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + return input_flags.eth ? ODP_PROTO_L2_TYPE_ETH : ODP_PROTO_L2_TYPE_NONE; +} + +_ODP_INLINE odp_proto_l3_type_t odp_packet_l3_type(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + if (input_flags.ipv4) + return ODP_PROTO_L3_TYPE_IPV4; + else if (input_flags.ipv6) + return ODP_PROTO_L3_TYPE_IPV6; + else if (input_flags.arp) + return ODP_PROTO_L3_TYPE_ARP; + + return ODP_PROTO_L3_TYPE_NONE; +} + +_ODP_INLINE odp_proto_l4_type_t odp_packet_l4_type(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + if (input_flags.tcp) + return ODP_PROTO_L4_TYPE_TCP; + else if (input_flags.udp) + return ODP_PROTO_L4_TYPE_UDP; + else if (input_flags.sctp) + return ODP_PROTO_L4_TYPE_SCTP; + else if (input_flags.ipsec_ah) + return ODP_PROTO_L4_TYPE_AH; + else if (input_flags.ipsec_esp) + return ODP_PROTO_L4_TYPE_ESP; + else if (input_flags.icmp && input_flags.ipv4) + return ODP_PROTO_L4_TYPE_ICMPV4; + else if (input_flags.icmp && input_flags.ipv6) + return ODP_PROTO_L4_TYPE_ICMPV6; + else if (input_flags.no_next_hdr) + return ODP_PROTO_L4_TYPE_NO_NEXT; + + return ODP_PROTO_L4_TYPE_NONE; +} + +_ODP_INLINE odp_packet_chksum_status_t odp_packet_l3_chksum_status(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + _odp_packet_input_flags_t input_flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + if (!input_flags.l3_chksum_done) + return ODP_PACKET_CHKSUM_UNKNOWN; + + if (flags.l3_chksum_err) + return ODP_PACKET_CHKSUM_BAD; + + return ODP_PACKET_CHKSUM_OK; +} + +_ODP_INLINE odp_packet_chksum_status_t odp_packet_l4_chksum_status(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + _odp_packet_input_flags_t input_flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + if (!input_flags.l4_chksum_done) + return ODP_PACKET_CHKSUM_UNKNOWN; + + if (flags.l4_chksum_err) + return ODP_PACKET_CHKSUM_BAD; + + return ODP_PACKET_CHKSUM_OK; +} + +_ODP_INLINE void odp_packet_l3_chksum_insert(odp_packet_t pkt, int insert) +{ + _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + + flags->l3_chksum_set = 1; + flags->l3_chksum = !!insert; +} + +_ODP_INLINE void odp_packet_l4_chksum_insert(odp_packet_t pkt, int insert) +{ + _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + + flags->l4_chksum_set = 1; + flags->l4_chksum = !!insert; +} + _ODP_INLINE uint32_t odp_packet_flow_hash(odp_packet_t pkt) { return _odp_pkt_get(pkt, uint32_t, rss); @@ -319,6 +481,23 @@ _ODP_INLINE odp_time_t odp_packet_ts(odp_packet_t pkt) return _odp_pkt_get(pkt, odp_time_t, timestamp); } +_ODP_INLINE void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) +{ + odp_time_t *ts = _odp_pkt_get_ptr(pkt, odp_time_t, timestamp); + _odp_packet_input_flags_t *input_flags = _odp_pkt_get_ptr(pkt, _odp_packet_input_flags_t, + input_flags); + + *ts = timestamp; + input_flags->timestamp = 1; +} + +_ODP_INLINE void odp_packet_ts_request(odp_packet_t pkt, int enable) +{ + _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + + flags->ts_set = !!enable; +} + _ODP_INLINE void *odp_packet_head(odp_packet_t pkt) { return (uint8_t *)odp_packet_data(pkt) - odp_packet_headroom(pkt); @@ -440,6 +619,8 @@ _ODP_INLINE int odp_packet_copy_from_mem(odp_packet_t pkt, uint32_t offset, _ODP_INLINE odp_packet_t odp_packet_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); + return (odp_packet_t)ev; } @@ -474,6 +655,8 @@ _ODP_INLINE odp_event_subtype_t odp_packet_subtype(odp_packet_t pkt) _ODP_INLINE odp_packet_tx_compl_t odp_packet_tx_compl_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET_TX_COMPL); + return (odp_packet_tx_compl_t)(uintptr_t)ev; } @@ -548,6 +731,15 @@ _ODP_INLINE int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt) return (int8_t)flags.shaper_len_adj; } +_ODP_INLINE uint64_t odp_packet_cls_mark(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + return input_flags.cls_mark ? _odp_pkt_get(pkt, uint16_t, cls_mark) : 0; +} + _ODP_INLINE uint32_t odp_packet_buf_data_len(odp_packet_buf_t pkt_buf) { return odp_packet_seg_data_len(ODP_PACKET_INVALID, (odp_packet_seg_t)pkt_buf); diff --git a/platform/linux-dpdk/include/odp/api/plat/rwlock_inlines.h b/platform/linux-dpdk/include/odp/api/plat/rwlock_inlines.h new file mode 120000 index 000000000..9b8d33f6a --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/rwlock_inlines.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/rwlock_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/rwlock_recursive_inlines.h b/platform/linux-dpdk/include/odp/api/plat/rwlock_recursive_inlines.h new file mode 120000 index 000000000..1b37e53d2 --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/rwlock_recursive_inlines.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/rwlock_recursive_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/schedule_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/schedule_inline_types.h new file mode 120000 index 000000000..a9b051e5b --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/schedule_inline_types.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/schedule_inline_types.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/schedule_inlines.h b/platform/linux-dpdk/include/odp/api/plat/schedule_inlines.h new file mode 120000 index 000000000..a15dbee6a --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/schedule_inlines.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/schedule_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/thread_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/thread_inline_types.h new file mode 120000 index 000000000..8f59ca214 --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/thread_inline_types.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/thread_inline_types.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h b/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h index 48154a26f..f8ff5c938 100644 --- a/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h @@ -7,9 +7,10 @@ #ifndef ODP_PLAT_TIMER_INLINES_H_ #define ODP_PLAT_TIMER_INLINES_H_ -#include <odp/api/event_types.h> +#include <odp/api/event.h> #include <odp/api/timer_types.h> +#include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/timer_inline_types.h> #include <stdint.h> @@ -53,6 +54,8 @@ _ODP_INLINE void *odp_timeout_user_area(odp_timeout_t tmo) _ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_TIMEOUT); + return (odp_timeout_t)ev; } diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h b/platform/linux-dpdk/include/odp_buffer_internal.h index dfffdc2be..b82f6af55 100644 --- a/platform/linux-dpdk/include/odp_buffer_internal.h +++ b/platform/linux-dpdk/include/odp_buffer_internal.h @@ -57,24 +57,6 @@ typedef struct ODP_ALIGNED_CACHE odp_buffer_hdr_t { } odp_buffer_hdr_t; -/* - * Buffer type - * - * @param buf Buffer handle - * - * @return Buffer type - */ -int _odp_buffer_type(odp_buffer_t buf); - -/* - * Buffer type set - * - * @param buf Buffer handle - * @param type New type value - * - */ -void _odp_buffer_type_set(odp_buffer_t buf, int type); - static inline struct rte_mbuf *_odp_buf_to_mbuf(odp_buffer_t buf) { return (struct rte_mbuf *)(uintptr_t)buf; diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h index ea700214d..141ee0af0 100644 --- a/platform/linux-dpdk/include/odp_packet_internal.h +++ b/platform/linux-dpdk/include/odp_packet_internal.h @@ -312,8 +312,8 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr, const uint32_t src_uarea_size = src_pool->params.pkt.uarea_size; const uint32_t dst_uarea_size = dst_pool->params.pkt.uarea_size; - ODP_ASSERT(dst_hdr->uarea_addr != NULL); - ODP_ASSERT(dst_uarea_size >= src_uarea_size); + _ODP_ASSERT(dst_hdr->uarea_addr != NULL); + _ODP_ASSERT(dst_uarea_size >= src_uarea_size); memcpy(dst_hdr->uarea_addr, src_hdr->uarea_addr, src_uarea_size); } else { @@ -321,7 +321,7 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr, /* If user area exists, packets should always be from the same pool, so * user area pointers can simply be swapped. */ - ODP_ASSERT(dst_hdr->event_hdr.pool == src_hdr->event_hdr.pool); + _ODP_ASSERT(dst_hdr->event_hdr.pool == src_hdr->event_hdr.pool); src_hdr->uarea_addr = dst_hdr->uarea_addr; dst_hdr->uarea_addr = src_uarea; @@ -377,11 +377,6 @@ static inline int packet_hdr_has_l2(odp_packet_hdr_t *pkt_hdr) return pkt_hdr->p.input_flags.l2; } -static inline void packet_hdr_has_l2_set(odp_packet_hdr_t *pkt_hdr, int val) -{ - pkt_hdr->p.input_flags.l2 = val; -} - static inline int packet_hdr_has_eth(odp_packet_hdr_t *pkt_hdr) { return pkt_hdr->p.input_flags.eth; diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h index c5a999fb4..27e7fa077 100644 --- a/platform/linux-dpdk/include/odp_packet_io_internal.h +++ b/platform/linux-dpdk/include/odp_packet_io_internal.h @@ -250,8 +250,8 @@ static inline pktio_entry_t *get_pktio_entry(odp_pktio_t pktio) return NULL; if (odp_unlikely(_odp_typeval(pktio) > ODP_CONFIG_PKTIO_ENTRIES)) { - ODP_DBG("pktio limit %" PRIuPTR "/%d exceed\n", - _odp_typeval(pktio), ODP_CONFIG_PKTIO_ENTRIES); + _ODP_DBG("pktio limit %" PRIuPTR "/%d exceed\n", + _odp_typeval(pktio), ODP_CONFIG_PKTIO_ENTRIES); return NULL; } diff --git a/platform/linux-dpdk/include/odp_ptr_ring_mpmc_internal.h b/platform/linux-dpdk/include/odp_ptr_ring_mpmc_internal.h index 9ebb47a9d..3e56c7f0e 100644 --- a/platform/linux-dpdk/include/odp_ptr_ring_mpmc_internal.h +++ b/platform/linux-dpdk/include/odp_ptr_ring_mpmc_internal.h @@ -48,8 +48,7 @@ static inline ring_mpmc_t ring_mpmc_create(const char *name, uint32_t size) rte_ring = rte_ring_create(ring_name, size, rte_socket_id(), 0); if (rte_ring == NULL) { - ODP_ERR("Creating DPDK ring failed: %s\n", - rte_strerror(rte_errno)); + _ODP_ERR("Creating DPDK ring failed: %s\n", rte_strerror(rte_errno)); return NULL; } diff --git a/platform/linux-dpdk/include/odp_ptr_ring_spsc_internal.h b/platform/linux-dpdk/include/odp_ptr_ring_spsc_internal.h index 85ddf80ea..3680f087a 100644 --- a/platform/linux-dpdk/include/odp_ptr_ring_spsc_internal.h +++ b/platform/linux-dpdk/include/odp_ptr_ring_spsc_internal.h @@ -55,8 +55,7 @@ static inline ring_spsc_t ring_spsc_create(const char *name, uint32_t size) rte_ring = rte_ring_create(ring_name, size, rte_socket_id(), RING_F_SP_ENQ | RING_F_SC_DEQ); if (rte_ring == NULL) { - ODP_ERR("Creating DPDK ring failed: %s\n", - rte_strerror(rte_errno)); + _ODP_ERR("Creating DPDK ring failed: %s\n", rte_strerror(rte_errno)); return NULL; } diff --git a/platform/linux-dpdk/include/odp_ptr_ring_st_internal.h b/platform/linux-dpdk/include/odp_ptr_ring_st_internal.h index 73982243b..cc258aeb4 100644 --- a/platform/linux-dpdk/include/odp_ptr_ring_st_internal.h +++ b/platform/linux-dpdk/include/odp_ptr_ring_st_internal.h @@ -48,8 +48,7 @@ static inline ring_st_t ring_st_create(const char *name, uint32_t size) rte_ring = rte_ring_create(ring_name, size, rte_socket_id(), RING_F_SP_ENQ | RING_F_SC_DEQ); if (rte_ring == NULL) { - ODP_ERR("Creating DPDK ring failed: %s\n", - rte_strerror(rte_errno)); + _ODP_ERR("Creating DPDK ring failed: %s\n", rte_strerror(rte_errno)); return NULL; } diff --git a/platform/linux-dpdk/odp_buffer.c b/platform/linux-dpdk/odp_buffer.c index 758782563..6cabe41e2 100644 --- a/platform/linux-dpdk/odp_buffer.c +++ b/platform/linux-dpdk/odp_buffer.c @@ -5,47 +5,28 @@ * SPDX-License-Identifier: BSD-3-Clause */ +#include <odp/api/align.h> #include <odp/api/buffer.h> +#include <odp/api/plat/buffer_inline_types.h> + #include <odp_buffer_internal.h> #include <odp_debug_internal.h> #include <odp_pool_internal.h> +#include <odp_print_internal.h> #include <string.h> #include <stdio.h> #include <inttypes.h> -int _odp_buffer_type(odp_buffer_t buf) -{ - odp_buffer_hdr_t *hdr = _odp_buf_hdr(buf); - - return hdr->event_hdr.type; -} - -void _odp_buffer_type_set(odp_buffer_t buf, int type) -{ - odp_buffer_hdr_t *hdr = _odp_buf_hdr(buf); +#include <odp/visibility_begin.h> - hdr->event_hdr.type = type; -} +/* Buffer header field offsets for inline functions */ +const _odp_buffer_inline_offset_t _odp_buffer_inline_offset ODP_ALIGNED_CACHE = { + .uarea_addr = offsetof(odp_buffer_hdr_t, uarea_addr) +}; -int odp_buffer_is_valid(odp_buffer_t buf) -{ - if (odp_event_is_valid(odp_buffer_to_event(buf)) == 0) - return 0; - - if (odp_event_type(odp_buffer_to_event(buf)) != ODP_EVENT_BUFFER) - return 0; - - return 1; -} - -void *odp_buffer_user_area(odp_buffer_t buf) -{ - odp_buffer_hdr_t *hdr = _odp_buf_hdr(buf); - - return hdr->uarea_addr; -} +#include <odp/visibility_end.h> void odp_buffer_print(odp_buffer_t buf) { @@ -57,21 +38,24 @@ void odp_buffer_print(odp_buffer_t buf) char str[max_len]; if (!odp_buffer_is_valid(buf)) { - ODP_ERR("Buffer is not valid.\n"); + _ODP_ERR("Buffer is not valid.\n"); return; } hdr = _odp_buf_hdr(buf); pool = _odp_pool_entry(hdr->event_hdr.pool); - len += snprintf(&str[len], n - len, "Buffer\n------\n"); - len += snprintf(&str[len], n - len, " pool index %u\n", pool->pool_idx); - len += snprintf(&str[len], n - len, " buffer index %u\n", hdr->event_hdr.index); - len += snprintf(&str[len], n - len, " addr %p\n", odp_buffer_addr(buf)); - len += snprintf(&str[len], n - len, " size %u\n", odp_buffer_size(buf)); + len += _odp_snprint(&str[len], n - len, "Buffer\n------\n"); + len += _odp_snprint(&str[len], n - len, " handle 0x%" PRIx64 "\n", + odp_buffer_to_u64(buf)); + len += _odp_snprint(&str[len], n - len, " pool index %u\n", pool->pool_idx); + len += _odp_snprint(&str[len], n - len, " buffer index %u\n", hdr->event_hdr.index); + len += _odp_snprint(&str[len], n - len, " addr %p\n", odp_buffer_addr(buf)); + len += _odp_snprint(&str[len], n - len, " size %u\n", odp_buffer_size(buf)); + len += _odp_snprint(&str[len], n - len, " user area %p\n", hdr->uarea_addr); str[len] = 0; - ODP_PRINT("\n%s\n", str); + _ODP_PRINT("%s\n", str); } uint64_t odp_buffer_to_u64(odp_buffer_t hdl) diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 0ed324ac1..076de1b0e 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -328,7 +328,7 @@ int _odp_crypto_init_global(void) odp_shm_t shm; if (odp_global_ro.disable.crypto) { - ODP_PRINT("\nODP crypto is DISABLED\n"); + _ODP_PRINT("\nODP crypto is DISABLED\n"); return 0; } @@ -342,11 +342,11 @@ int _odp_crypto_init_global(void) if (shm != ODP_SHM_INVALID) { global = odp_shm_addr(shm); if (global == NULL) { - ODP_ERR("Failed to find the reserved shm block"); + _ODP_ERR("Failed to find the reserved shm block"); return -1; } } else { - ODP_ERR("Shared memory reserve failed.\n"); + _ODP_ERR("Shared memory reserve failed.\n"); return -1; } @@ -368,7 +368,7 @@ int _odp_crypto_init_global(void) cdev_count = rte_cryptodev_count(); if (cdev_count == 0) { - ODP_PRINT("No crypto devices available\n"); + _ODP_PRINT("No crypto devices available\n"); return 0; } @@ -389,7 +389,7 @@ int _odp_crypto_init_global(void) if (nb_queue_pairs > dev_info.max_nb_queue_pairs) { nb_queue_pairs = dev_info.max_nb_queue_pairs; queue_pairs_shared = true; - ODP_PRINT("Using shared queue pairs for crypto device %" + _ODP_PRINT("Using shared queue pairs for crypto device %" PRIu16 " (driver: %s)\n", cdev_id, dev_info.driver_name); } @@ -429,20 +429,18 @@ int _odp_crypto_init_global(void) 0, socket_id); if (mp == NULL) { - ODP_ERR("Cannot create session pool on socket %d\n", - socket_id); + _ODP_ERR("Cannot create session pool on socket %d\n", socket_id); return -1; } - ODP_PRINT("Allocated session pool on socket %d\n", - socket_id); + _ODP_PRINT("Allocated session pool on socket %d\n", socket_id); global->session_mempool[socket_id] = mp; } mp = global->session_mempool[socket_id]; rc = rte_cryptodev_configure(cdev_id, &conf); if (rc < 0) { - ODP_ERR("Failed to configure cryptodev %u", cdev_id); + _ODP_ERR("Failed to configure cryptodev %u", cdev_id); return -1; } @@ -456,16 +454,15 @@ int _odp_crypto_init_global(void) &qp_conf, socket_id); if (rc < 0) { - ODP_ERR("Fail to setup queue pair %u on dev %u", - queue_pair, cdev_id); + _ODP_ERR("Fail to setup queue pair %u on dev %u", + queue_pair, cdev_id); return -1; } } rc = rte_cryptodev_start(cdev_id); if (rc < 0) { - ODP_ERR("Failed to start device %u: error %d\n", - cdev_id, rc); + _ODP_ERR("Failed to start device %u: error %d\n", cdev_id, rc); return -1; } @@ -495,7 +492,7 @@ int _odp_crypto_init_global(void) rte_socket_id()); if (global->crypto_op_pool == NULL) { - ODP_ERR("Cannot create crypto op pool\n"); + _ODP_ERR("Cannot create crypto op pool\n"); return -1; } @@ -619,7 +616,7 @@ int odp_crypto_capability(odp_crypto_capability_t *capability) uint8_t cdev_id, cdev_count; if (odp_global_ro.disable.crypto) { - ODP_ERR("Crypto is disabled\n"); + _ODP_ERR("Crypto is disabled\n"); return -1; } @@ -631,7 +628,7 @@ int odp_crypto_capability(odp_crypto_capability_t *capability) cdev_count = rte_cryptodev_count(); if (cdev_count == 0) { - ODP_ERR("No crypto devices available\n"); + _ODP_ERR("No crypto devices available\n"); return 0; } @@ -746,7 +743,7 @@ static int cipher_aead_capability(odp_cipher_alg_t cipher, cdev_count = rte_cryptodev_count(); if (cdev_count == 0) { - ODP_ERR("No crypto devices available\n"); + _ODP_ERR("No crypto devices available\n"); return -1; } @@ -798,7 +795,7 @@ static int cipher_capability(odp_cipher_alg_t cipher, cdev_count = rte_cryptodev_count(); if (cdev_count == 0) { - ODP_ERR("No crypto devices available\n"); + _ODP_ERR("No crypto devices available\n"); return -1; } @@ -958,7 +955,7 @@ static int auth_aead_capability(odp_auth_alg_t auth, cdev_count = rte_cryptodev_count(); if (cdev_count == 0) { - ODP_ERR("No crypto devices available\n"); + _ODP_ERR("No crypto devices available\n"); return -1; } @@ -1045,7 +1042,7 @@ static int auth_capability(odp_auth_alg_t auth, cdev_count = rte_cryptodev_count(); if (cdev_count == 0) { - ODP_ERR("No crypto devices available\n"); + _ODP_ERR("No crypto devices available\n"); return -1; } @@ -1129,7 +1126,7 @@ static int get_crypto_aead_dev(struct rte_crypto_sym_xform *aead_xform, /* Check if key size is supported by the algorithm. */ if (is_valid_size(aead_xform->aead.key.length, &cap->sym.aead.key_size) != 0) { - ODP_ERR("Unsupported aead key length\n"); + _ODP_ERR("Unsupported aead key length\n"); continue; } @@ -1137,14 +1134,14 @@ static int get_crypto_aead_dev(struct rte_crypto_sym_xform *aead_xform, if (aead_xform->aead.iv.length > MAX_IV_LENGTH || is_valid_size(aead_xform->aead.iv.length, &cap->sym.aead.iv_size) != 0) { - ODP_ERR("Unsupported iv length\n"); + _ODP_ERR("Unsupported iv length\n"); continue; } /* Check if digest size is supported by the algorithm. */ if (is_valid_size(aead_xform->aead.digest_length, &cap->sym.aead.digest_size) != 0) { - ODP_ERR("Unsupported digest length\n"); + _ODP_ERR("Unsupported digest length\n"); continue; } @@ -1183,7 +1180,7 @@ static int get_crypto_dev(struct rte_crypto_sym_xform *cipher_xform, /* Check if key size is supported by the algorithm. */ if (is_valid_size(cipher_xform->cipher.key.length, &cap->sym.cipher.key_size) != 0) { - ODP_ERR("Unsupported cipher key length\n"); + _ODP_ERR("Unsupported cipher key length\n"); continue; } @@ -1191,7 +1188,7 @@ static int get_crypto_dev(struct rte_crypto_sym_xform *cipher_xform, if (cipher_xform->cipher.iv.length > MAX_IV_LENGTH || is_valid_size(cipher_xform->cipher.iv.length, &cap->sym.cipher.iv_size) != 0) { - ODP_ERR("Unsupported iv length\n"); + _ODP_ERR("Unsupported iv length\n"); continue; } @@ -1227,14 +1224,14 @@ check_auth: /* Check if key size is supported by the algorithm. */ if (is_valid_size(auth_xform->auth.key.length, &cap->sym.auth.key_size) != 0) { - ODP_ERR("Unsupported auth key length\n"); + _ODP_ERR("Unsupported auth key length\n"); continue; } /* Check if digest size is supported by the algorithm. */ if (is_valid_size(auth_xform->auth.digest_length, &cap->sym.auth.digest_size) != 0) { - ODP_ERR("Unsupported digest length\n"); + _ODP_ERR("Unsupported digest length\n"); continue; } @@ -1242,7 +1239,7 @@ check_auth: if (auth_xform->auth.iv.length > MAX_IV_LENGTH || is_valid_size(auth_xform->auth.iv.length, &cap->sym.auth.iv_size) != 0) { - ODP_ERR("Unsupported iv length\n"); + _ODP_ERR("Unsupported iv length\n"); continue; } @@ -1316,7 +1313,7 @@ static int crypto_fill_auth_xform(struct rte_crypto_sym_xform *auth_xform, auth_xform->auth.digest_length = param->auth_digest_len; if (auth_xform->auth.digest_length > PACKET_DIGEST_MAX) { - ODP_ERR("Requested too long digest\n"); + _ODP_ERR("Requested too long digest\n"); return -1; } @@ -1349,20 +1346,20 @@ static int crypto_fill_aead_xform(struct rte_crypto_sym_xform *aead_xform, aead_xform->aead.aad_length = param->auth_aad_len; if (aead_xform->aead.aad_length > PACKET_AAD_MAX) { - ODP_ERR("Requested too long AAD\n"); + _ODP_ERR("Requested too long AAD\n"); return -1; } if (aead_xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM && aead_xform->aead.aad_length + AES_CCM_AAD_OFFSET > PACKET_AAD_MAX) { - ODP_ERR("Requested too long AAD for CCM\n"); + _ODP_ERR("Requested too long AAD for CCM\n"); return -1; } aead_xform->aead.digest_length = param->auth_digest_len; if (aead_xform->aead.digest_length > PACKET_DIGEST_MAX) { - ODP_ERR("Requested too long digest\n"); + _ODP_ERR("Requested too long digest\n"); return -1; } @@ -1390,7 +1387,7 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, crypto_session_entry_t *session = NULL; if (odp_global_ro.disable.crypto) { - ODP_ERR("Crypto is disabled\n"); + _ODP_ERR("Crypto is disabled\n"); /* Dummy output to avoid compiler warning about uninitialized * variables */ *status = ODP_CRYPTO_SES_ERR_ENOMEM; @@ -1399,7 +1396,7 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, } if (rte_cryptodev_count() == 0) { - ODP_ERR("No crypto devices available\n"); + _ODP_ERR("No crypto devices available\n"); *status = ODP_CRYPTO_SES_ERR_ENOMEM; goto err; } @@ -1407,7 +1404,7 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, /* Allocate memory for this session */ session = alloc_session(); if (session == NULL) { - ODP_ERR("Failed to allocate a session session"); + _ODP_ERR("Failed to allocate a session session"); *status = ODP_CRYPTO_SES_ERR_ENOMEM; goto err; } @@ -1468,7 +1465,7 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, &cdev_id); } if (rc) { - ODP_ERR("Couldn't find a crypto device"); + _ODP_ERR("Couldn't find a crypto device"); *status = ODP_CRYPTO_SES_ERR_ENOMEM; goto err; } @@ -1563,7 +1560,7 @@ int _odp_crypto_term_global(void) for (session = global->free; session != NULL; session = session->next) count++; if (count != MAX_SESSIONS) { - ODP_ERR("crypto sessions still active\n"); + _ODP_ERR("crypto sessions still active\n"); rc = -1; } @@ -1572,7 +1569,7 @@ int _odp_crypto_term_global(void) ret = odp_shm_free(global->shm); if (ret < 0) { - ODP_ERR("shm free failed for crypto_pool\n"); + _ODP_ERR("shm free failed for crypto_pool\n"); rc = -1; } @@ -1584,7 +1581,7 @@ odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev) { /* This check not mandated by the API specification */ if (odp_event_type(ev) != ODP_EVENT_CRYPTO_COMPL) - ODP_ABORT("Event not a crypto completion"); + _ODP_ABORT("Event not a crypto completion"); return (odp_crypto_compl_t)ev; } @@ -1600,7 +1597,7 @@ void odp_crypto_compl_result(odp_crypto_compl_t completion_event, (void)result; /* We won't get such events anyway, so there can be no result */ - ODP_ASSERT(0); + _ODP_ASSERT(0); } void odp_crypto_compl_free(odp_crypto_compl_t completion_event) @@ -1629,8 +1626,8 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) odp_packet_t odp_crypto_packet_from_event(odp_event_t ev) { /* This check not mandated by the API specification */ - ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); - ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); + _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); return odp_packet_from_event(ev); } @@ -1653,8 +1650,7 @@ int odp_crypto_result(odp_crypto_packet_result_t *result, { odp_crypto_packet_result_t *op_result; - ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == - ODP_EVENT_PACKET_CRYPTO); + _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == ODP_EVENT_PACKET_CRYPTO); op_result = get_op_result_from_packet(packet); @@ -1728,9 +1724,9 @@ static void crypto_fill_aead_param(const crypto_session_entry_t *session, else if (session->p.cipher_iv.data) memcpy(iv_ptr, session->cipher_iv_data, iv_len); else - ODP_ASSERT(iv_len == 0); + _ODP_ASSERT(iv_len == 0); #else - ODP_ASSERT(iv_len == 0 || param->cipher_iv_ptr != NULL); + _ODP_ASSERT(iv_len == 0 || param->cipher_iv_ptr != NULL); memcpy(iv_ptr, param->cipher_iv_ptr, iv_len); #endif @@ -1764,7 +1760,7 @@ static void crypto_fill_sym_param(const crypto_session_entry_t *session, iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET); memcpy(iv_ptr, session->cipher_iv_data, cipher_iv_len); } else { - ODP_ASSERT(cipher_iv_len == 0); + _ODP_ASSERT(cipher_iv_len == 0); } if (param->auth_iv_ptr) { @@ -1776,11 +1772,11 @@ static void crypto_fill_sym_param(const crypto_session_entry_t *session, IV_OFFSET + MAX_IV_LENGTH); memcpy(iv_ptr, session->auth_iv_data, auth_iv_len); } else { - ODP_ASSERT(auth_iv_len == 0); + _ODP_ASSERT(auth_iv_len == 0); } #else - ODP_ASSERT(cipher_iv_len == 0 || param->cipher_iv_ptr != NULL); - ODP_ASSERT(auth_iv_len == 0 || param->auth_iv_ptr != NULL); + _ODP_ASSERT(cipher_iv_len == 0 || param->cipher_iv_ptr != NULL); + _ODP_ASSERT(auth_iv_len == 0 || param->auth_iv_ptr != NULL); iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET); memcpy(iv_ptr, param->cipher_iv_ptr, cipher_iv_len); @@ -1836,7 +1832,7 @@ static int linearize_pkt(const crypto_session_entry_t *session, odp_packet_t pkt odp_packet_move_data(pkt, 0, shift, len); /* We rely on our trunc implementation to not change the handle */ rc = odp_packet_trunc_tail(&pkt, shift, NULL, NULL); - ODP_ASSERT(rc == 0); + _ODP_ASSERT(rc == 0); return odp_packet_num_segs(pkt) != 1; } @@ -1849,13 +1845,13 @@ static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src) md_copy = _odp_packet_copy_md_possible(odp_packet_pool(dst), odp_packet_pool(src)); if (odp_unlikely(md_copy < 0)) { - ODP_ERR("Unable to copy packet metadata\n"); + _ODP_ERR("Unable to copy packet metadata\n"); return -1; } rc = odp_packet_copy_from_pkt(dst, 0, src, 0, odp_packet_len(src)); if (odp_unlikely(rc < 0)) { - ODP_ERR("Unable to copy packet data\n"); + _ODP_ERR("Unable to copy packet data\n"); return -1; } @@ -1875,7 +1871,7 @@ static odp_packet_t get_output_packet(const crypto_session_entry_t *session, if (pkt_out == ODP_PACKET_INVALID) { odp_pool_t pool = session->p.output_pool; - ODP_ASSERT(pool != ODP_POOL_INVALID); + _ODP_ASSERT(pool != ODP_POOL_INVALID); if (pool == odp_packet_pool(pkt_in)) { pkt_out = pkt_in; } else { @@ -1910,11 +1906,11 @@ static int op_alloc(crypto_op_t *op[], (struct rte_crypto_op **)op, num_pkts) == 0)) { /* This should not happen since we made op pool big enough */ - ODP_DBG("falling back to single crypto op alloc\n"); + _ODP_DBG("falling back to single crypto op alloc\n"); op[0] = (crypto_op_t *)rte_crypto_op_alloc(global->crypto_op_pool, RTE_CRYPTO_OP_TYPE_SYMMETRIC); if (odp_unlikely(op[0] == NULL)) { - ODP_ERR("Failed to allocate crypto operation\n"); + _ODP_ERR("Failed to allocate crypto operation\n"); return 0; } num_pkts = 1; @@ -1924,7 +1920,7 @@ static int op_alloc(crypto_op_t *op[], odp_packet_t pkt; session = (crypto_session_entry_t *)(intptr_t)param[n].session; - ODP_ASSERT(session != NULL); + _ODP_ASSERT(session != NULL); pkt = get_output_packet(session, pkt_in[n], pkt_out[n]); if (odp_unlikely(pkt == ODP_PACKET_INVALID)) { @@ -2014,7 +2010,7 @@ static void dev_enq_deq(uint8_t cdev_id, int thread_id, crypto_op_t *op[], int n */ for (int n = rc; n < num_op; n++) op[n]->state.status = S_ERROR; - ODP_ERR("Failed to enqueue crypto operations\n"); + _ODP_ERR("Failed to enqueue crypto operations\n"); num_op = rc; if (num_op == 0) return; @@ -2035,7 +2031,7 @@ static void dev_enq_deq(uint8_t cdev_id, int thread_id, crypto_op_t *op[], int n if (odp_unlikely(rc == 0)) { odp_time_wait_ns(DEQ_RETRY_DELAY_NS); if (++retry_count == MAX_DEQ_RETRIES) { - ODP_ERR("Failed to dequeue crypto operations\n"); + _ODP_ERR("Failed to dequeue crypto operations\n"); /* * We cannot give up and return to the caller * since some packets and crypto operations @@ -2049,8 +2045,8 @@ static void dev_enq_deq(uint8_t cdev_id, int thread_id, crypto_op_t *op[], int n odp_spinlock_unlock(&global->lock); for (int n = 0; n < num_dequeued; n++) { - ODP_ASSERT((crypto_op_t *)deq_op[n] == op[n]); - ODP_ASSERT((odp_packet_t)deq_op[n]->sym->m_src == op[n]->state.pkt); + _ODP_ASSERT((crypto_op_t *)deq_op[n] == op[n]); + _ODP_ASSERT((odp_packet_t)deq_op[n]->sym->m_src == op[n]->state.pkt); } } @@ -2249,7 +2245,7 @@ int odp_crypto_op(const odp_packet_t pkt_in[], for (i = 0; i < num_pkt; i++) { session = (crypto_session_entry_t *)(intptr_t)param[i].session; - ODP_ASSERT(ODP_CRYPTO_SYNC == session->p.op_mode); + _ODP_ASSERT(ODP_CRYPTO_SYNC == session->p.op_mode); } return odp_crypto_int(pkt_in, pkt_out, param, num_pkt); } @@ -2269,8 +2265,8 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[], for (i = 0; i < num_pkt; i++) { session = (crypto_session_entry_t *)(intptr_t)param[i].session; - ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); - ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); + _ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); + _ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); out_pkts[i] = pkt_out[i]; } diff --git a/platform/linux-dpdk/odp_errno.c b/platform/linux-dpdk/odp_errno.c index c30cb0039..2f03ee898 100644 --- a/platform/linux-dpdk/odp_errno.c +++ b/platform/linux-dpdk/odp_errno.c @@ -23,9 +23,9 @@ void odp_errno_zero(void) void odp_errno_print(const char *str) { if (str != NULL) - ODP_PRINT("%s %s\n", str, strerror(rte_errno)); + _ODP_PRINT("%s %s\n", str, strerror(rte_errno)); else - ODP_PRINT("%s\n", strerror(rte_errno)); + _ODP_PRINT("%s\n", strerror(rte_errno)); } const char *odp_errno_str(int errnum) diff --git a/platform/linux-dpdk/odp_event.c b/platform/linux-dpdk/odp_event.c index 940c16e9d..24853f83e 100644 --- a/platform/linux-dpdk/odp_event.c +++ b/platform/linux-dpdk/odp_event.c @@ -72,7 +72,7 @@ void odp_event_free(odp_event_t event) odp_dma_compl_free(odp_dma_compl_from_event(event)); break; default: - ODP_ABORT("Invalid event type: %d\n", odp_event_type(event)); + _ODP_ABORT("Invalid event type: %d\n", odp_event_type(event)); } } diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c index 3ae981b68..329252f9c 100644 --- a/platform/linux-dpdk/odp_init.c +++ b/platform/linux-dpdk/odp_init.c @@ -9,16 +9,19 @@ #include <odp/api/init.h> #include <odp/api/shared_memory.h> + +#include <odp/api/plat/thread_inlines.h> + #include <odp_debug_internal.h> +#include <odp_global_data.h> #include <odp_init_internal.h> -#include <odp_schedule_if.h> #include <odp_libconfig_internal.h> -#include <odp/api/plat/thread_inlines.h> +#include <odp_schedule_if.h> #include <odp_shm_internal.h> + #include <string.h> #include <stdio.h> #include <unistd.h> -#include <ctype.h> #include <inttypes.h> #include <rte_config.h> @@ -59,6 +62,15 @@ enum init_stage { odp_global_data_ro_t odp_global_ro; odp_global_data_rw_t *odp_global_rw; +/* Global function pointers for inline header usage. The values are written + * during odp_init_global() (enables process mode support). */ +#include <odp/visibility_begin.h> + +odp_log_func_t ODP_PRINTF_FORMAT(2, 3) _odp_log_fn; +odp_abort_func_t _odp_abort_fn; + +#include <odp/visibility_end.h> + /* odp_init_local() call status */ static __thread uint8_t init_local_called; @@ -124,7 +136,7 @@ static int read_pci_config(char **pci_cmd) /* Buffer to concatenate list of '-w/-b <pci addr>' strings */ buf = malloc(pci_count * (str_size + 3)); if (buf == NULL) { - ODP_ERR("PCI config buffer alloc fail\n"); + _ODP_ERR("PCI config buffer alloc fail\n"); return -1; } @@ -136,7 +148,7 @@ static int read_pci_config(char **pci_cmd) strcat(buf, addr_str); } - ODP_PRINT(" %s: %s\n\n", pci_str[i], buf); + _ODP_PRINT(" %s: %s\n\n", pci_str[i], buf); /* No need to read blacklist if whitelist is defined */ *pci_cmd = buf; @@ -158,7 +170,7 @@ static int read_eal_cmdstr(char **eal_cmd) buf = malloc(length); if (buf == NULL) { - ODP_ERR("DPDK EAL command string buffer alloc fail\n"); + _ODP_ERR("DPDK EAL command string buffer alloc fail\n"); return -1; } @@ -167,7 +179,7 @@ static int read_eal_cmdstr(char **eal_cmd) return -1; } - ODP_PRINT(" %s: %s\n\n", dpdk_str, buf); + _ODP_PRINT(" %s: %s\n\n", dpdk_str, buf); *eal_cmd = buf; return length; @@ -182,16 +194,16 @@ static int _odp_init_dpdk(const char *cmdline) int val = 0; char *pci_cmd = NULL, *eal_cmd = NULL; - ODP_PRINT("DPDK config:\n"); + _ODP_PRINT("DPDK config:\n"); str = "dpdk.process_mode_memory_mb"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } mem_prealloc = val; - ODP_PRINT(" %s: %" PRIu32 "\n", str, mem_prealloc); + _ODP_PRINT(" %s: %" PRIu32 "\n", str, mem_prealloc); if (cmdline == NULL) { cmdline = getenv("ODP_PLATFORM_PARAMS"); @@ -201,14 +213,14 @@ static int _odp_init_dpdk(const char *cmdline) pcicmdlen = read_pci_config(&pci_cmd); if (pcicmdlen < 0) { - ODP_ERR("Error reading PCI config\n"); + _ODP_ERR("Error reading PCI config\n"); return -1; } /* Read any additional EAL command string from config */ ealcmdlen = read_eal_cmdstr(&eal_cmd); if (ealcmdlen < 0) { - ODP_ERR("Error reading additional DPDK EAL command string\n"); + _ODP_ERR("Error reading additional DPDK EAL command string\n"); if (pci_cmd != NULL) free(pci_cmd); return -1; @@ -249,18 +261,18 @@ static int _odp_init_dpdk(const char *cmdline) dpdk_argc = rte_strsplit(full_cmdline, strlen(full_cmdline), dpdk_argv, dpdk_argc, ' '); for (i = 0; i < dpdk_argc; ++i) - ODP_DBG("arg[%d]: %s\n", i, dpdk_argv[i]); + _ODP_DBG("arg[%d]: %s\n", i, dpdk_argv[i]); fflush(stdout); i = rte_eal_init(dpdk_argc, dpdk_argv); if (i < 0) { - ODP_ERR("Cannot init the Intel DPDK EAL!\n"); + _ODP_ERR("Cannot init the Intel DPDK EAL!\n"); return -1; } else if (i + 1 != dpdk_argc) { - ODP_DBG("Some DPDK args were not processed!\n"); - ODP_DBG("Passed: %d Consumed %d\n", dpdk_argc, i + 1); + _ODP_DBG("Some DPDK args were not processed!\n"); + _ODP_DBG("Passed: %d Consumed %d\n", dpdk_argc, i + 1); } - ODP_DBG("rte_eal_init OK\n"); + _ODP_DBG("rte_eal_init OK\n"); /* Reset to 0 to force getopt() internal initialization routine */ optind = 0; @@ -283,7 +295,7 @@ static int global_rw_data_init(void) odp_global_rw = odp_shm_addr(shm); if (odp_global_rw == NULL) { - ODP_ERR("Global RW data shm reserve failed.\n"); + _ODP_ERR("Global RW data shm reserve failed.\n"); return -1; } @@ -298,12 +310,12 @@ static int global_rw_data_term(void) shm = odp_shm_lookup("_odp_global_rw_data"); if (shm == ODP_SHM_INVALID) { - ODP_ERR("Unable to find global RW data shm.\n"); + _ODP_ERR("Unable to find global RW data shm.\n"); return -1; } if (odp_shm_free(shm)) { - ODP_ERR("Global RW data shm free failed.\n"); + _ODP_ERR("Global RW data shm free failed.\n"); return -1; } @@ -318,126 +330,126 @@ static int term_global(enum init_stage stage) case ALL_INIT: case DMA_INIT: if (_odp_dma_term_global()) { - ODP_ERR("ODP DMA term failed.\n"); + _ODP_ERR("ODP DMA term failed.\n"); rc = -1; } /* Fall through */ case IPSEC_INIT: if (_odp_ipsec_term_global()) { - ODP_ERR("ODP IPsec term failed.\n"); + _ODP_ERR("ODP IPsec term failed.\n"); rc = -1; } /* Fall through */ case IPSEC_SAD_INIT: if (_odp_ipsec_sad_term_global()) { - ODP_ERR("ODP IPsec SAD term failed.\n"); + _ODP_ERR("ODP IPsec SAD term failed.\n"); rc = -1; } /* Fall through */ case IPSEC_EVENTS_INIT: if (_odp_ipsec_events_term_global()) { - ODP_ERR("ODP IPsec events term failed.\n"); + _ODP_ERR("ODP IPsec events term failed.\n"); rc = -1; } /* Fall through */ case NAME_TABLE_INIT: if (_odp_int_name_tbl_term_global()) { - ODP_ERR("Name table term failed.\n"); + _ODP_ERR("Name table term failed.\n"); rc = -1; } /* Fall through */ case TRAFFIC_MNGR_INIT: if (_odp_tm_term_global()) { - ODP_ERR("TM term failed.\n"); + _ODP_ERR("TM term failed.\n"); rc = -1; } /* Fall through */ case CLASSIFICATION_INIT: if (_odp_classification_term_global()) { - ODP_ERR("ODP classification term failed.\n"); + _ODP_ERR("ODP classification term failed.\n"); rc = -1; } /* Fall through */ case COMP_INIT: if (_odp_comp_term_global()) { - ODP_ERR("ODP comp term failed.\n"); + _ODP_ERR("ODP comp term failed.\n"); rc = -1; } /* Fall through */ case CRYPTO_INIT: if (_odp_crypto_term_global()) { - ODP_ERR("ODP crypto term failed.\n"); + _ODP_ERR("ODP crypto term failed.\n"); rc = -1; } /* Fall through */ case TIMER_INIT: if (_odp_timer_term_global()) { - ODP_ERR("ODP timer term failed.\n"); + _ODP_ERR("ODP timer term failed.\n"); rc = -1; } /* Fall through */ case PKTIO_INIT: if (_odp_pktio_term_global()) { - ODP_ERR("ODP pktio term failed.\n"); + _ODP_ERR("ODP pktio term failed.\n"); rc = -1; } /* Fall through */ case SCHED_INIT: if (_odp_schedule_term_global()) { - ODP_ERR("ODP schedule term failed.\n"); + _ODP_ERR("ODP schedule term failed.\n"); rc = -1; } /* Fall through */ case QUEUE_INIT: if (_odp_queue_term_global()) { - ODP_ERR("ODP queue term failed.\n"); + _ODP_ERR("ODP queue term failed.\n"); rc = -1; } /* Fall through */ case STASH_INIT: if (_odp_stash_term_global()) { - ODP_ERR("ODP stash term failed.\n"); + _ODP_ERR("ODP stash term failed.\n"); rc = -1; } /* Fall through */ case POOL_INIT: if (_odp_pool_term_global()) { - ODP_ERR("ODP buffer pool term failed.\n"); + _ODP_ERR("ODP buffer pool term failed.\n"); rc = -1; } /* Fall through */ case THREAD_INIT: if (_odp_thread_term_global()) { - ODP_ERR("ODP thread term failed.\n"); + _ODP_ERR("ODP thread term failed.\n"); rc = -1; } /* Fall through */ case HASH_INIT: if (_odp_hash_term_global()) { - ODP_ERR("ODP hash term failed.\n"); + _ODP_ERR("ODP hash term failed.\n"); rc = -1; } /* Fall through */ case GLOBAL_RW_DATA_INIT: if (global_rw_data_term()) { - ODP_ERR("ODP global RW data term failed.\n"); + _ODP_ERR("ODP global RW data term failed.\n"); rc = -1; } /* Fall through */ @@ -446,21 +458,21 @@ static int term_global(enum init_stage stage) case FDSERVER_INIT: case ISHM_INIT: if (_odp_shm_term_global()) { - ODP_ERR("ODP shm term failed.\n"); + _ODP_ERR("ODP shm term failed.\n"); rc = -1; } /* Fall through */ case SYSINFO_INIT: if (_odp_system_info_term()) { - ODP_ERR("ODP system info term failed.\n"); + _ODP_ERR("ODP system info term failed.\n"); rc = -1; } /* Fall through */ case TIME_INIT: if (_odp_time_term_global()) { - ODP_ERR("ODP time term failed.\n"); + _ODP_ERR("ODP time term failed.\n"); rc = -1; } /* Fall through */ @@ -469,14 +481,14 @@ static int term_global(enum init_stage stage) /* Fall through */ case CPUMASK_INIT: if (_odp_cpumask_term_global()) { - ODP_ERR("ODP cpumask term failed.\n"); + _ODP_ERR("ODP cpumask term failed.\n"); rc = -1; } /* Fall through */ case LIBCONFIG_INIT: if (_odp_libconfig_term_global()) { - ODP_ERR("ODP runtime config term failed.\n"); + _ODP_ERR("ODP runtime config term failed.\n"); rc = -1; } /* Fall through */ @@ -496,24 +508,23 @@ int odp_init_global(odp_instance_t *instance, memset(&odp_global_ro, 0, sizeof(odp_global_data_ro_t)); odp_global_ro.main_pid = getpid(); - - odp_global_ro.log_fn = odp_override_log; - odp_global_ro.abort_fn = odp_override_abort; + _odp_log_fn = odp_override_log; + _odp_abort_fn = odp_override_abort; odp_init_param_init(&odp_global_ro.init_param); if (params != NULL) { odp_global_ro.init_param = *params; if (params->log_fn != NULL) - odp_global_ro.log_fn = params->log_fn; + _odp_log_fn = params->log_fn; if (params->abort_fn != NULL) - odp_global_ro.abort_fn = params->abort_fn; + _odp_abort_fn = params->abort_fn; if (params->mem_model == ODP_MEM_MODEL_PROCESS) odp_global_ro.shm_single_va = 1; } if (_odp_libconfig_init_global()) { - ODP_ERR("ODP runtime config init failed.\n"); + _ODP_ERR("ODP runtime config init failed.\n"); goto init_failed; } stage = LIBCONFIG_INIT; @@ -521,144 +532,144 @@ int odp_init_global(odp_instance_t *instance, disable_features(&odp_global_ro, params); if (_odp_cpumask_init_global(params)) { - ODP_ERR("ODP cpumask init failed.\n"); + _ODP_ERR("ODP cpumask init failed.\n"); goto init_failed; } stage = CPUMASK_INIT; if (_odp_cpu_cycles_init_global()) { - ODP_ERR("ODP cpu cycle init failed.\n"); + _ODP_ERR("ODP cpu cycle init failed.\n"); goto init_failed; } stage = CPU_CYCLES_INIT; if (_odp_init_dpdk((const char *)platform_params)) { - ODP_ERR("ODP dpdk init failed.\n"); + _ODP_ERR("ODP dpdk init failed.\n"); return -1; } if (_odp_time_init_global()) { - ODP_ERR("ODP time init failed.\n"); + _ODP_ERR("ODP time init failed.\n"); goto init_failed; } stage = TIME_INIT; if (_odp_system_info_init()) { - ODP_ERR("ODP system_info init failed.\n"); + _ODP_ERR("ODP system_info init failed.\n"); goto init_failed; } stage = SYSINFO_INIT; if (_odp_shm_init_global(params)) { - ODP_ERR("ODP shm init failed.\n"); + _ODP_ERR("ODP shm init failed.\n"); goto init_failed; } stage = ISHM_INIT; if (global_rw_data_init()) { - ODP_ERR("ODP global RW data init failed.\n"); + _ODP_ERR("ODP global RW data init failed.\n"); goto init_failed; } stage = GLOBAL_RW_DATA_INIT; if (_odp_hash_init_global()) { - ODP_ERR("ODP hash init failed.\n"); + _ODP_ERR("ODP hash init failed.\n"); goto init_failed; } stage = HASH_INIT; if (_odp_thread_init_global()) { - ODP_ERR("ODP thread init failed.\n"); + _ODP_ERR("ODP thread init failed.\n"); goto init_failed; } stage = THREAD_INIT; if (_odp_pool_init_global()) { - ODP_ERR("ODP pool init failed.\n"); + _ODP_ERR("ODP pool init failed.\n"); goto init_failed; } stage = POOL_INIT; if (_odp_stash_init_global()) { - ODP_ERR("ODP stash init failed.\n"); + _ODP_ERR("ODP stash init failed.\n"); goto init_failed; } stage = STASH_INIT; if (_odp_queue_init_global()) { - ODP_ERR("ODP queue init failed.\n"); + _ODP_ERR("ODP queue init failed.\n"); goto init_failed; } stage = QUEUE_INIT; if (_odp_schedule_init_global()) { - ODP_ERR("ODP schedule init failed.\n"); + _ODP_ERR("ODP schedule init failed.\n"); goto init_failed; } stage = SCHED_INIT; if (_odp_pktio_init_global()) { - ODP_ERR("ODP packet io init failed.\n"); + _ODP_ERR("ODP packet io init failed.\n"); goto init_failed; } stage = PKTIO_INIT; if (_odp_timer_init_global(params)) { - ODP_ERR("ODP timer init failed.\n"); + _ODP_ERR("ODP timer init failed.\n"); goto init_failed; } stage = TIMER_INIT; if (_odp_crypto_init_global()) { - ODP_ERR("ODP crypto init failed.\n"); + _ODP_ERR("ODP crypto init failed.\n"); goto init_failed; } stage = CRYPTO_INIT; if (_odp_comp_init_global()) { - ODP_ERR("ODP comp init failed.\n"); + _ODP_ERR("ODP comp init failed.\n"); goto init_failed; } stage = COMP_INIT; if (_odp_classification_init_global()) { - ODP_ERR("ODP classification init failed.\n"); + _ODP_ERR("ODP classification init failed.\n"); goto init_failed; } stage = CLASSIFICATION_INIT; if (_odp_tm_init_global()) { - ODP_ERR("ODP traffic manager init failed\n"); + _ODP_ERR("ODP traffic manager init failed\n"); goto init_failed; } stage = TRAFFIC_MNGR_INIT; if (_odp_int_name_tbl_init_global()) { - ODP_ERR("ODP name table init failed\n"); + _ODP_ERR("ODP name table init failed\n"); goto init_failed; } stage = NAME_TABLE_INIT; if (_odp_ipsec_events_init_global()) { - ODP_ERR("ODP IPsec events init failed.\n"); + _ODP_ERR("ODP IPsec events init failed.\n"); goto init_failed; } stage = IPSEC_EVENTS_INIT; if (_odp_ipsec_sad_init_global()) { - ODP_ERR("ODP IPsec SAD init failed.\n"); + _ODP_ERR("ODP IPsec SAD init failed.\n"); goto init_failed; } stage = IPSEC_SAD_INIT; if (_odp_ipsec_init_global()) { - ODP_ERR("ODP IPsec init failed.\n"); + _ODP_ERR("ODP IPsec init failed.\n"); goto init_failed; } stage = IPSEC_INIT; if (_odp_dma_init_global()) { - ODP_ERR("ODP DMA init failed.\n"); + _ODP_ERR("ODP DMA init failed.\n"); goto init_failed; } stage = DMA_INIT; @@ -676,7 +687,7 @@ init_failed: int odp_term_global(odp_instance_t instance) { if (instance != (odp_instance_t)odp_global_ro.main_pid) { - ODP_ERR("Bad instance.\n"); + _ODP_ERR("Bad instance.\n"); return -1; } return term_global(ALL_INIT); @@ -692,42 +703,42 @@ static int term_local(enum init_stage stage) case SCHED_INIT: if (_odp_sched_fn->term_local()) { - ODP_ERR("ODP schedule local term failed.\n"); + _ODP_ERR("ODP schedule local term failed.\n"); rc = -1; } /* Fall through */ case QUEUE_INIT: if (_odp_queue_fn->term_local()) { - ODP_ERR("ODP queue local term failed.\n"); + _ODP_ERR("ODP queue local term failed.\n"); rc = -1; } /* Fall through */ case POOL_INIT: if (_odp_pool_term_local()) { - ODP_ERR("ODP buffer pool local term failed.\n"); + _ODP_ERR("ODP buffer pool local term failed.\n"); rc = -1; } /* Fall through */ case CRYPTO_INIT: if (_odp_crypto_term_local()) { - ODP_ERR("ODP crypto local term failed.\n"); + _ODP_ERR("ODP crypto local term failed.\n"); rc = -1; } /* Fall through */ case RANDOM_INIT: if (_odp_random_term_local()) { - ODP_ERR("ODP random local term failed.\n"); + _ODP_ERR("ODP random local term failed.\n"); rc = -1; } /* Fall through */ case TIMER_INIT: if (_odp_timer_term_local()) { - ODP_ERR("ODP timer local term failed.\n"); + _ODP_ERR("ODP timer local term failed.\n"); rc = -1; } /* Fall through */ @@ -735,7 +746,7 @@ static int term_local(enum init_stage stage) case THREAD_INIT: rc_thd = _odp_thread_term_local(); if (rc_thd < 0) { - ODP_ERR("ODP thread local term failed.\n"); + _ODP_ERR("ODP thread local term failed.\n"); rc = -1; } else { if (!rc) @@ -745,7 +756,7 @@ static int term_local(enum init_stage stage) case ISHM_INIT: if (_odp_shm_term_local()) { - ODP_ERR("ODP shm local term failed.\n"); + _ODP_ERR("ODP shm local term failed.\n"); rc = -1; } /* Fall through */ @@ -762,67 +773,67 @@ int odp_init_local(odp_instance_t instance, odp_thread_type_t thr_type) enum init_stage stage = NO_INIT; if (instance != (odp_instance_t)odp_global_ro.main_pid) { - ODP_ERR("Bad instance.\n"); + _ODP_ERR("Bad instance.\n"); goto init_fail; } /* Detect if odp_init_local() has been already called from this thread */ if (getpid() == odp_global_ro.main_pid && init_local_called) { - ODP_ERR("%s() called multiple times by the same thread\n", __func__); + _ODP_ERR("%s() called multiple times by the same thread\n", __func__); goto init_fail; } init_local_called = 1; if (_odp_shm_init_local()) { - ODP_ERR("ODP shm local init failed.\n"); + _ODP_ERR("ODP shm local init failed.\n"); goto init_fail; } stage = ISHM_INIT; if (_odp_thread_init_local(thr_type)) { - ODP_ERR("ODP thread local init failed.\n"); + _ODP_ERR("ODP thread local init failed.\n"); goto init_fail; } stage = THREAD_INIT; if (_odp_pktio_init_local()) { - ODP_ERR("ODP packet io local init failed.\n"); + _ODP_ERR("ODP packet io local init failed.\n"); goto init_fail; } stage = PKTIO_INIT; if (_odp_timer_init_local()) { - ODP_ERR("ODP timer local init failed.\n"); + _ODP_ERR("ODP timer local init failed.\n"); goto init_fail; } stage = TIMER_INIT; if (_odp_random_init_local()) { - ODP_ERR("ODP random local init failed.\n"); + _ODP_ERR("ODP random local init failed.\n"); goto init_fail; } stage = RANDOM_INIT; if (_odp_crypto_init_local()) { - ODP_ERR("ODP crypto local init failed.\n"); + _ODP_ERR("ODP crypto local init failed.\n"); goto init_fail; } stage = CRYPTO_INIT; if (_odp_pool_init_local()) { - ODP_ERR("ODP pool local init failed.\n"); + _ODP_ERR("ODP pool local init failed.\n"); goto init_fail; } stage = POOL_INIT; if (_odp_queue_fn->init_local()) { - ODP_ERR("ODP queue local init failed.\n"); + _ODP_ERR("ODP queue local init failed.\n"); goto init_fail; } stage = QUEUE_INIT; if (_odp_sched_fn->init_local()) { - ODP_ERR("ODP schedule local init failed.\n"); + _ODP_ERR("ODP schedule local init failed.\n"); goto init_fail; } /* stage = SCHED_INIT; */ @@ -838,7 +849,7 @@ int odp_term_local(void) { /* Check that odp_init_local() has been called by this thread */ if (!init_local_called) { - ODP_ERR("%s() called by a non-initialized thread\n", __func__); + _ODP_ERR("%s() called by a non-initialized thread\n", __func__); return -1; } init_local_called = 0; diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index 99dbe07f6..83be3dfed 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -29,6 +29,7 @@ #include <odp_packet_io_internal.h> #include <odp_parse_internal.h> #include <odp_pool_internal.h> +#include <odp_print_internal.h> #include <rte_version.h> @@ -63,6 +64,7 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .input_flags = offsetof(odp_packet_hdr_t, p.input_flags), .flags = offsetof(odp_packet_hdr_t, p.flags), .subtype = offsetof(odp_packet_hdr_t, subtype), + .cls_mark = offsetof(odp_packet_hdr_t, cls_mark), .buf_addr = offsetof(odp_packet_hdr_t, event_hdr.mb.buf_addr), .data = offsetof(odp_packet_hdr_t, event_hdr.mb.data_off), .pkt_len = offsetof(odp_packet_hdr_t, event_hdr.mb.pkt_len), @@ -291,7 +293,7 @@ odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len) { pool_t *pool = _odp_pool_entry(pool_hdl); - ODP_ASSERT(pool->type == ODP_POOL_PACKET); + _ODP_ASSERT(pool->type == ODP_POOL_PACKET); if (odp_unlikely(len == 0)) return ODP_PACKET_INVALID; @@ -304,7 +306,7 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, { pool_t *pool = _odp_pool_entry(pool_hdl); - ODP_ASSERT(pool->type == ODP_POOL_PACKET); + _ODP_ASSERT(pool->type == ODP_POOL_PACKET); if (odp_unlikely(len == 0)) return -1; @@ -318,9 +320,8 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) return -1; if (RTE_PKTMBUF_HEADROOM + len > odp_packet_buf_len(pkt)) { - ODP_DBG("Not enough head room for that packet %d/%d\n", - RTE_PKTMBUF_HEADROOM + len, - odp_packet_buf_len(pkt)); + _ODP_DBG("Not enough head room for that packet %d/%d\n", + RTE_PKTMBUF_HEADROOM + len, odp_packet_buf_len(pkt)); return -1; } @@ -613,53 +614,6 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, void **tail_ptr, * */ -void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ptr) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (odp_unlikely(ptr == NULL)) { - pkt_hdr->p.flags.user_ptr_set = 0; - return; - } - - pkt_hdr->user_ptr = ptr; - pkt_hdr->p.flags.user_ptr_set = 1; -} - -int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (odp_unlikely(offset >= (odp_packet_len(pkt) - 1))) - return -1; - - packet_hdr_has_l2_set(pkt_hdr, 1); - pkt_hdr->p.l2_offset = offset; - return 0; -} - -int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (odp_unlikely(offset >= (odp_packet_len(pkt) - 1))) - return -1; - - pkt_hdr->p.l3_offset = offset; - return 0; -} - -int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (odp_unlikely(offset >= (odp_packet_len(pkt) - 1))) - return -1; - - pkt_hdr->p.l4_offset = offset; - return 0; -} - uint16_t odp_packet_ones_comp(odp_packet_t pkt, odp_packet_data_range_t *range) { (void)pkt; @@ -668,55 +622,6 @@ uint16_t odp_packet_ones_comp(odp_packet_t pkt, odp_packet_data_range_t *range) return 0; } -void odp_packet_l3_chksum_insert(odp_packet_t pkt, int insert) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - pkt_hdr->p.flags.l3_chksum_set = 1; - pkt_hdr->p.flags.l3_chksum = insert; -} - -void odp_packet_l4_chksum_insert(odp_packet_t pkt, int insert) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - pkt_hdr->p.flags.l4_chksum_set = 1; - pkt_hdr->p.flags.l4_chksum = insert; -} - -odp_packet_chksum_status_t odp_packet_l3_chksum_status(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (!pkt_hdr->p.input_flags.l3_chksum_done) - return ODP_PACKET_CHKSUM_UNKNOWN; - - if (pkt_hdr->p.flags.l3_chksum_err) - return ODP_PACKET_CHKSUM_BAD; - - return ODP_PACKET_CHKSUM_OK; -} - -odp_packet_chksum_status_t odp_packet_l4_chksum_status(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (!pkt_hdr->p.input_flags.l4_chksum_done) - return ODP_PACKET_CHKSUM_UNKNOWN; - - if (pkt_hdr->p.flags.l4_chksum_err) - return ODP_PACKET_CHKSUM_BAD; - - return ODP_PACKET_CHKSUM_OK; -} - -void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - packet_set_ts(pkt_hdr, ×tamp); -} - /* * * Manipulation @@ -904,7 +809,7 @@ odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_pool_t pool) odp_packet_t newpkt; if (odp_unlikely(_odp_packet_copy_md_possible(pool, odp_packet_pool(pkt)) < 0)) { - ODP_ERR("Unable to copy packet metadata\n"); + _ODP_ERR("Unable to copy packet metadata\n"); return ODP_PACKET_INVALID; } @@ -1038,7 +943,7 @@ int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset, uint32_t cmplen; int ret; - ODP_ASSERT(offset + len <= odp_packet_len(pkt)); + _ODP_ASSERT(offset + len <= odp_packet_len(pkt)); while (len > 0) { mapaddr = odp_packet_offset(pkt, offset, &seglen, NULL); @@ -1065,31 +970,31 @@ static int packet_print_input_flags(odp_packet_hdr_t *hdr, char *str, int max) int len = 0; if (hdr->p.input_flags.l2) - len += snprintf(&str[len], max - len, "l2 "); + len += _odp_snprint(&str[len], max - len, "l2 "); if (hdr->p.input_flags.l3) - len += snprintf(&str[len], max - len, "l3 "); + len += _odp_snprint(&str[len], max - len, "l3 "); if (hdr->p.input_flags.l4) - len += snprintf(&str[len], max - len, "l4 "); + len += _odp_snprint(&str[len], max - len, "l4 "); if (hdr->p.input_flags.eth) - len += snprintf(&str[len], max - len, "eth "); + len += _odp_snprint(&str[len], max - len, "eth "); if (hdr->p.input_flags.vlan) - len += snprintf(&str[len], max - len, "vlan "); + len += _odp_snprint(&str[len], max - len, "vlan "); if (hdr->p.input_flags.arp) - len += snprintf(&str[len], max - len, "arp "); + len += _odp_snprint(&str[len], max - len, "arp "); if (hdr->p.input_flags.ipv4) - len += snprintf(&str[len], max - len, "ipv4 "); + len += _odp_snprint(&str[len], max - len, "ipv4 "); if (hdr->p.input_flags.ipv6) - len += snprintf(&str[len], max - len, "ipv6 "); + len += _odp_snprint(&str[len], max - len, "ipv6 "); if (hdr->p.input_flags.ipsec) - len += snprintf(&str[len], max - len, "ipsec "); + len += _odp_snprint(&str[len], max - len, "ipsec "); if (hdr->p.input_flags.udp) - len += snprintf(&str[len], max - len, "udp "); + len += _odp_snprint(&str[len], max - len, "udp "); if (hdr->p.input_flags.tcp) - len += snprintf(&str[len], max - len, "tcp "); + len += _odp_snprint(&str[len], max - len, "tcp "); if (hdr->p.input_flags.sctp) - len += snprintf(&str[len], max - len, "sctp "); + len += _odp_snprint(&str[len], max - len, "sctp "); if (hdr->p.input_flags.icmp) - len += snprintf(&str[len], max - len, "icmp "); + len += _odp_snprint(&str[len], max - len, "icmp "); return len; } @@ -1104,56 +1009,59 @@ void odp_packet_print(odp_packet_t pkt) odp_packet_hdr_t *hdr = packet_hdr(pkt); pool_t *pool = _odp_pool_entry(hdr->event_hdr.pool); - len += snprintf(&str[len], n - len, "Packet\n------\n"); - len += snprintf(&str[len], n - len, " pool index %u\n", pool->pool_idx); - len += snprintf(&str[len], n - len, " buf index %u\n", hdr->event_hdr.index); - len += snprintf(&str[len], n - len, " ev subtype %i\n", hdr->subtype); - len += snprintf(&str[len], n - len, " input_flags 0x%" PRIx64 "\n", - hdr->p.input_flags.all); + len += _odp_snprint(&str[len], n - len, "Packet info\n"); + len += _odp_snprint(&str[len], n - len, "-----------\n"); + len += _odp_snprint(&str[len], n - len, " handle 0x%" PRIx64 "\n", + odp_packet_to_u64(pkt)); + len += _odp_snprint(&str[len], n - len, " pool index %u\n", pool->pool_idx); + len += _odp_snprint(&str[len], n - len, " buf index %u\n", hdr->event_hdr.index); + len += _odp_snprint(&str[len], n - len, " ev subtype %i\n", hdr->subtype); + len += _odp_snprint(&str[len], n - len, " input_flags 0x%" PRIx64 "\n", + hdr->p.input_flags.all); if (hdr->p.input_flags.all) { - len += snprintf(&str[len], n - len, " "); + len += _odp_snprint(&str[len], n - len, " "); len += packet_print_input_flags(hdr, &str[len], n - len); - len += snprintf(&str[len], n - len, "\n"); + len += _odp_snprint(&str[len], n - len, "\n"); } - len += snprintf(&str[len], n - len, " flags 0x%" PRIx32 "\n", - hdr->p.flags.all_flags); - len += snprintf(&str[len], n - len, " cls_mark %" PRIu64 "\n", - odp_packet_cls_mark(pkt)); - len += snprintf(&str[len], n - len, - " l2_offset %" PRIu32 "\n", hdr->p.l2_offset); - len += snprintf(&str[len], n - len, - " l3_offset %" PRIu32 "\n", hdr->p.l3_offset); - len += snprintf(&str[len], n - len, - " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); - len += snprintf(&str[len], n - len, - " frame_len %" PRIu32 "\n", - hdr->event_hdr.mb.pkt_len); - len += snprintf(&str[len], n - len, - " input %" PRIu64 "\n", - odp_pktio_to_u64(hdr->input)); - len += snprintf(&str[len], n - len, - " headroom %" PRIu32 "\n", - odp_packet_headroom(pkt)); - len += snprintf(&str[len], n - len, - " tailroom %" PRIu32 "\n", - odp_packet_tailroom(pkt)); - len += snprintf(&str[len], n - len, - " num_segs %i\n", odp_packet_num_segs(pkt)); + len += _odp_snprint(&str[len], n - len, + " flags 0x%" PRIx32 "\n", hdr->p.flags.all_flags); + len += _odp_snprint(&str[len], n - len, + " cls_mark %" PRIu64 "\n", odp_packet_cls_mark(pkt)); + len += _odp_snprint(&str[len], n - len, + " user ptr %p\n", hdr->user_ptr); + len += _odp_snprint(&str[len], n - len, + " user area %p\n", hdr->uarea_addr); + len += _odp_snprint(&str[len], n - len, + " l2_offset %" PRIu32 "\n", hdr->p.l2_offset); + len += _odp_snprint(&str[len], n - len, + " l3_offset %" PRIu32 "\n", hdr->p.l3_offset); + len += _odp_snprint(&str[len], n - len, + " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); + len += _odp_snprint(&str[len], n - len, + " frame_len %" PRIu32 "\n", hdr->event_hdr.mb.pkt_len); + len += _odp_snprint(&str[len], n - len, + " input %" PRIu64 "\n", odp_pktio_to_u64(hdr->input)); + len += _odp_snprint(&str[len], n - len, + " headroom %" PRIu32 "\n", odp_packet_headroom(pkt)); + len += _odp_snprint(&str[len], n - len, + " tailroom %" PRIu32 "\n", odp_packet_tailroom(pkt)); + len += _odp_snprint(&str[len], n - len, + " num_segs %i\n", odp_packet_num_segs(pkt)); seg = odp_packet_first_seg(pkt); for (int seg_idx = 0; seg != ODP_PACKET_SEG_INVALID; seg_idx++) { - len += snprintf(&str[len], n - len, - " [%d] seg_len %-4" PRIu32 " seg_data %p\n", - seg_idx, odp_packet_seg_data_len(pkt, seg), - odp_packet_seg_data(pkt, seg)); + len += _odp_snprint(&str[len], n - len, + " [%d] seg_len %-4" PRIu32 " seg_data %p\n", + seg_idx, odp_packet_seg_data_len(pkt, seg), + odp_packet_seg_data(pkt, seg)); seg = odp_packet_next_seg(pkt, seg); } str[len] = '\0'; - ODP_PRINT("\n%s\n", str); + _ODP_PRINT("%s\n", str); } void odp_packet_print_data(odp_packet_t pkt, uint32_t offset, @@ -1169,26 +1077,28 @@ void odp_packet_print_data(odp_packet_t pkt, uint32_t offset, uint32_t data_len = odp_packet_len(pkt); pool_t *pool = _odp_pool_entry(hdr->event_hdr.pool); - len += snprintf(&str[len], n - len, "Packet\n------\n"); - len += snprintf(&str[len], n - len, - " pool name %s\n", pool->name); - len += snprintf(&str[len], n - len, - " buf index %" PRIu32 "\n", hdr->event_hdr.index); - len += snprintf(&str[len], n - len, - " segcount %" PRIu8 "\n", - hdr->event_hdr.mb.nb_segs); - len += snprintf(&str[len], n - len, - " data len %" PRIu32 "\n", data_len); - len += snprintf(&str[len], n - len, - " data ptr %p\n", odp_packet_data(pkt)); - len += snprintf(&str[len], n - len, - " print offset %" PRIu32 "\n", offset); - len += snprintf(&str[len], n - len, - " print length %" PRIu32 "\n", byte_len); + len += _odp_snprint(&str[len], n - len, "Packet data\n"); + len += _odp_snprint(&str[len], n - len, "-----------\n"); + len += _odp_snprint(&str[len], n - len, + " handle 0x%" PRIx64 "\n", odp_packet_to_u64(pkt)); + len += _odp_snprint(&str[len], n - len, + " pool name %s\n", pool->name); + len += _odp_snprint(&str[len], n - len, + " buf index %" PRIu32 "\n", hdr->event_hdr.index); + len += _odp_snprint(&str[len], n - len, + " segcount %" PRIu8 "\n", hdr->event_hdr.mb.nb_segs); + len += _odp_snprint(&str[len], n - len, + " data len %" PRIu32 "\n", data_len); + len += _odp_snprint(&str[len], n - len, + " data ptr %p\n", odp_packet_data(pkt)); + len += _odp_snprint(&str[len], n - len, + " print offset %" PRIu32 "\n", offset); + len += _odp_snprint(&str[len], n - len, + " print length %" PRIu32 "\n", byte_len); if (offset + byte_len > data_len) { - len += snprintf(&str[len], n - len, " BAD OFFSET OR LEN\n"); - ODP_PRINT("%s\n", str); + len += _odp_snprint(&str[len], n - len, " BAD OFFSET OR LEN\n"); + _ODP_PRINT("%s\n", str); return; } @@ -1204,18 +1114,18 @@ void odp_packet_print_data(odp_packet_t pkt, uint32_t offset, odp_packet_copy_to_mem(pkt, offset, copy_len, data); - len += snprintf(&str[len], n - len, " "); + len += _odp_snprint(&str[len], n - len, " "); for (i = 0; i < copy_len; i++) - len += snprintf(&str[len], n - len, " %02x", data[i]); + len += _odp_snprint(&str[len], n - len, " %02x", data[i]); - len += snprintf(&str[len], n - len, "\n"); + len += _odp_snprint(&str[len], n - len, "\n"); byte_len -= copy_len; offset += copy_len; } - ODP_PRINT("%s\n", str); + _ODP_PRINT("%s\n", str); } int odp_packet_is_valid(odp_packet_t pkt) @@ -1521,7 +1431,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, if (sum != 0) { pkt_hdr->p.flags.l4_chksum_err = 1; pkt_hdr->p.flags.udp_err = 1; - ODP_DBG("UDP chksum fail (%x)!\n", sum); + _ODP_DBG("UDP chksum fail (%x)!\n", sum); if (opt.bit.drop_udp_err) return -1; } @@ -1541,7 +1451,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, if (sum != 0) { pkt_hdr->p.flags.l4_chksum_err = 1; pkt_hdr->p.flags.tcp_err = 1; - ODP_DBG("TCP chksum fail (%x)!\n", sum); + _ODP_DBG("TCP chksum fail (%x)!\n", sum); if (opt.bit.drop_tcp_err) return -1; } @@ -1573,8 +1483,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, if (sum != sctp->chksum) { pkt_hdr->p.flags.l4_chksum_err = 1; pkt_hdr->p.flags.sctp_err = 1; - ODP_DBG("SCTP chksum fail (%x/%x)!\n", sum, - sctp->chksum); + _ODP_DBG("SCTP chksum fail (%x/%x)!\n", sum, sctp->chksum); if (opt.bit.drop_sctp_err) return -1; } @@ -1589,6 +1498,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); const uint8_t *data; uint32_t seg_len; + uint32_t seg_end; uint32_t packet_len = odp_packet_len(pkt); odp_proto_t proto = param->proto; odp_proto_layer_t layer = param->last_layer; @@ -1621,6 +1531,8 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, data = buf; } + seg_end = offset + seg_len; /* one past the maximum offset */ + /* Reset parser flags, keep other flags */ packet_parse_reset(pkt_hdr, 0); @@ -1644,7 +1556,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, opt.bit.sctp_chksum = param->chksums.chksum.sctp; ret = _odp_packet_parse_common_l3_l4(&pkt_hdr->p, data, offset, - packet_len, seg_len, layer, + packet_len, seg_end, layer, ethtype, &l4_part_sum, opt); if (ret) @@ -1739,19 +1651,19 @@ odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) odp_packet_t new; int ret; - ODP_ASSERT(!odp_packet_has_ref(pkt)); + _ODP_ASSERT(!odp_packet_has_ref(pkt)); new = odp_packet_copy(pkt, odp_packet_pool(pkt)); if (new == ODP_PACKET_INVALID) { - ODP_ERR("copy failed\n"); + _ODP_ERR("copy failed\n"); return ODP_PACKET_INVALID; } ret = odp_packet_trunc_head(&new, offset, NULL, NULL); if (ret < 0) { - ODP_ERR("trunk_head failed\n"); + _ODP_ERR("trunk_head failed\n"); odp_packet_free(new); return ODP_PACKET_INVALID; } @@ -1765,12 +1677,12 @@ odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, odp_packet_t new; int ret; - ODP_ASSERT(!odp_packet_has_ref(pkt)); + _ODP_ASSERT(!odp_packet_has_ref(pkt)); new = odp_packet_copy(pkt, odp_packet_pool(pkt)); if (new == ODP_PACKET_INVALID) { - ODP_ERR("copy failed\n"); + _ODP_ERR("copy failed\n"); return ODP_PACKET_INVALID; } @@ -1778,7 +1690,7 @@ odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, ret = odp_packet_trunc_head(&new, offset, NULL, NULL); if (ret < 0) { - ODP_ERR("trunk_head failed\n"); + _ODP_ERR("trunk_head failed\n"); odp_packet_free(new); return ODP_PACKET_INVALID; } @@ -1787,7 +1699,7 @@ odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, ret = odp_packet_concat(&hdr, new); if (ret < 0) { - ODP_ERR("concat failed\n"); + _ODP_ERR("concat failed\n"); odp_packet_free(new); return ODP_PACKET_INVALID; } @@ -1795,73 +1707,6 @@ odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, return hdr; } -odp_proto_l2_type_t odp_packet_l2_type(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (pkt_hdr->p.input_flags.eth) - return ODP_PROTO_L2_TYPE_ETH; - - return ODP_PROTO_L2_TYPE_NONE; -} - -odp_proto_l3_type_t odp_packet_l3_type(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (pkt_hdr->p.input_flags.ipv4) - return ODP_PROTO_L3_TYPE_IPV4; - else if (pkt_hdr->p.input_flags.ipv6) - return ODP_PROTO_L3_TYPE_IPV6; - else if (pkt_hdr->p.input_flags.arp) - return ODP_PROTO_L3_TYPE_ARP; - - return ODP_PROTO_L3_TYPE_NONE; -} - -odp_proto_l4_type_t odp_packet_l4_type(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (pkt_hdr->p.input_flags.tcp) - return ODP_PROTO_L4_TYPE_TCP; - else if (pkt_hdr->p.input_flags.udp) - return ODP_PROTO_L4_TYPE_UDP; - else if (pkt_hdr->p.input_flags.sctp) - return ODP_PROTO_L4_TYPE_SCTP; - else if (pkt_hdr->p.input_flags.ipsec_ah) - return ODP_PROTO_L4_TYPE_AH; - else if (pkt_hdr->p.input_flags.ipsec_esp) - return ODP_PROTO_L4_TYPE_ESP; - else if (pkt_hdr->p.input_flags.icmp && - pkt_hdr->p.input_flags.ipv4) - return ODP_PROTO_L4_TYPE_ICMPV4; - else if (pkt_hdr->p.input_flags.icmp && - pkt_hdr->p.input_flags.ipv6) - return ODP_PROTO_L4_TYPE_ICMPV6; - else if (pkt_hdr->p.input_flags.no_next_hdr) - return ODP_PROTO_L4_TYPE_NO_NEXT; - - return ODP_PROTO_L4_TYPE_NONE; -} - -uint64_t odp_packet_cls_mark(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (pkt_hdr->p.input_flags.cls_mark) - return pkt_hdr->cls_mark; - - return 0; -} - -void odp_packet_ts_request(odp_packet_t pkt, int enable) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - pkt_hdr->p.flags.ts_set = !!enable; -} - void odp_packet_lso_request_clr(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); @@ -1931,7 +1776,7 @@ int odp_packet_has_tx_compl_request(odp_packet_t pkt) void odp_packet_tx_compl_free(odp_packet_tx_compl_t tx_compl) { if (odp_unlikely(tx_compl == ODP_PACKET_TX_COMPL_INVALID)) { - ODP_ERR("Bad TX completion event handle\n"); + _ODP_ERR("Bad TX completion event handle\n"); return; } @@ -1941,7 +1786,7 @@ void odp_packet_tx_compl_free(odp_packet_tx_compl_t tx_compl) void *odp_packet_tx_compl_user_ptr(odp_packet_tx_compl_t tx_compl) { if (odp_unlikely(tx_compl == ODP_PACKET_TX_COMPL_INVALID)) { - ODP_ERR("Bad TX completion event handle\n"); + _ODP_ERR("Bad TX completion event handle\n"); return NULL; } @@ -1982,18 +1827,17 @@ uint32_t odp_packet_disassemble(odp_packet_t pkt, odp_packet_buf_t pkt_buf[], uint32_t num_segs = odp_packet_num_segs(pkt); if (odp_unlikely(pool->type != ODP_POOL_PACKET)) { - ODP_ERR("Not a packet pool\n"); + _ODP_ERR("Not a packet pool\n"); return 0; } if (odp_unlikely(pool->pool_ext == 0)) { - ODP_ERR("Not an external memory pool\n"); + _ODP_ERR("Not an external memory pool\n"); return 0; } if (odp_unlikely(num < num_segs)) { - ODP_ERR("Not enough buffer handles %u. Packet has %u segments.\n", - num, num_segs); + _ODP_ERR("Not enough buffer handles %u. Packet has %u segments.\n", num, num_segs); return 0; } @@ -2018,17 +1862,17 @@ odp_packet_t odp_packet_reassemble(odp_pool_t pool_hdl, pool_t *pool = _odp_pool_entry(pool_hdl); if (odp_unlikely(pool->type != ODP_POOL_PACKET)) { - ODP_ERR("Not a packet pool\n"); + _ODP_ERR("Not a packet pool\n"); return ODP_PACKET_INVALID; } if (odp_unlikely(pool->pool_ext == 0)) { - ODP_ERR("Not an external memory pool\n"); + _ODP_ERR("Not an external memory pool\n"); return ODP_PACKET_INVALID; } if (odp_unlikely(num == 0)) { - ODP_ERR("Bad number of buffers: %u\n", num); + _ODP_ERR("Bad number of buffers: %u\n", num); return ODP_PACKET_INVALID; } diff --git a/platform/linux-dpdk/odp_packet_dpdk.c b/platform/linux-dpdk/odp_packet_dpdk.c index a40e907e1..a8e4298fb 100644 --- a/platform/linux-dpdk/odp_packet_dpdk.c +++ b/platform/linux-dpdk/odp_packet_dpdk.c @@ -163,8 +163,7 @@ static int lookup_opt(const char *opt_name, const char *drv_name, int *val) ret = _odp_libconfig_lookup_ext_int(base, drv_name, opt_name, val); if (ret == 0) - ODP_ERR("Unable to find DPDK configuration option: %s\n", - opt_name); + _ODP_ERR("Unable to find DPDK configuration option: %s\n", opt_name); return ret; } @@ -192,12 +191,12 @@ static int init_options(pktio_entry_t *pktio_entry, return -1; opt->multicast_enable = !!opt->multicast_enable; - ODP_DBG("DPDK interface (%s): %" PRIu16 "\n", dev_info->driver_name, - pkt_priv(pktio_entry)->port_id); - ODP_DBG(" multicast: %d\n", opt->multicast_enable); - ODP_DBG(" num_rx_desc: %d\n", opt->num_rx_desc); - ODP_DBG(" num_tx_desc: %d\n", opt->num_tx_desc_default); - ODP_DBG(" rx_drop_en: %d\n", opt->rx_drop_en); + _ODP_DBG("DPDK interface (%s): %" PRIu16 "\n", dev_info->driver_name, + pkt_priv(pktio_entry)->port_id); + _ODP_DBG(" multicast: %d\n", opt->multicast_enable); + _ODP_DBG(" num_rx_desc: %d\n", opt->num_rx_desc); + _ODP_DBG(" num_tx_desc: %d\n", opt->num_tx_desc_default); + _ODP_DBG(" rx_drop_en: %d\n", opt->rx_drop_en); return 0; } @@ -249,7 +248,7 @@ static int dpdk_maxlen_set(pktio_entry_t *pktio_entry, uint32_t maxlen_input, ret = rte_eth_dev_set_mtu(pkt_dpdk->port_id, mtu); if (odp_unlikely(ret)) - ODP_ERR("rte_eth_dev_set_mtu() failed: %d\n", ret); + _ODP_ERR("rte_eth_dev_set_mtu() failed: %d\n", ret); pkt_dpdk->mtu = maxlen_input; pkt_dpdk->mtu_set = 1; @@ -317,8 +316,8 @@ static int dpdk_setup_eth_dev(pktio_entry_t *pktio_entry) pktio_entry->num_in_queue, pktio_entry->num_out_queue, ð_conf); if (ret < 0) { - ODP_ERR("Failed to setup device: err=%d, port=%" PRIu8 "\n", - ret, pkt_dpdk->port_id); + _ODP_ERR("Failed to setup device: err=%d, port=%" PRIu8 "\n", + ret, pkt_dpdk->port_id); return -1; } return 0; @@ -330,15 +329,14 @@ static void _dpdk_print_port_mac(uint16_t port_id) memset(ð_addr, 0, sizeof(eth_addr)); rte_eth_macaddr_get(port_id, ð_addr); - ODP_DBG("Port %" PRIu16 ", " - "MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n", - port_id, - eth_addr.addr_bytes[0], - eth_addr.addr_bytes[1], - eth_addr.addr_bytes[2], - eth_addr.addr_bytes[3], - eth_addr.addr_bytes[4], - eth_addr.addr_bytes[5]); + _ODP_DBG("Port %" PRIu16 ", MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n", + port_id, + eth_addr.addr_bytes[0], + eth_addr.addr_bytes[1], + eth_addr.addr_bytes[2], + eth_addr.addr_bytes[3], + eth_addr.addr_bytes[4], + eth_addr.addr_bytes[5]); } static void prepare_rss_conf(pktio_entry_t *pktio_entry, @@ -360,33 +358,33 @@ static void prepare_rss_conf(pktio_entry_t *pktio_entry, /* Print debug info about unsupported hash protocols */ if (p->hash_proto.proto.ipv4 && ((rss_hf_capa & ETH_RSS_IPV4) == 0)) - ODP_PRINT("DPDK: hash_proto.ipv4 not supported (rss_hf_capa 0x%" PRIx64 ")\n", - rss_hf_capa); + _ODP_PRINT("DPDK: hash_proto.ipv4 not supported (rss_hf_capa 0x%" PRIx64 ")\n", + rss_hf_capa); if (p->hash_proto.proto.ipv4_udp && ((rss_hf_capa & ETH_RSS_NONFRAG_IPV4_UDP) == 0)) - ODP_PRINT("DPDK: hash_proto.ipv4_udp not supported (rss_hf_capa 0x%" PRIx64 ")\n", - rss_hf_capa); + _ODP_PRINT("DPDK: hash_proto.ipv4_udp not supported (rss_hf_capa 0x%" PRIx64 ")\n", + rss_hf_capa); if (p->hash_proto.proto.ipv4_tcp && ((rss_hf_capa & ETH_RSS_NONFRAG_IPV4_TCP) == 0)) - ODP_PRINT("DPDK: hash_proto.ipv4_tcp not supported (rss_hf_capa 0x%" PRIx64 ")\n", - rss_hf_capa); + _ODP_PRINT("DPDK: hash_proto.ipv4_tcp not supported (rss_hf_capa 0x%" PRIx64 ")\n", + rss_hf_capa); if (p->hash_proto.proto.ipv6 && ((rss_hf_capa & ETH_RSS_IPV6) == 0)) - ODP_PRINT("DPDK: hash_proto.ipv6 not supported (rss_hf_capa 0x%" PRIx64 ")\n", - rss_hf_capa); + _ODP_PRINT("DPDK: hash_proto.ipv6 not supported (rss_hf_capa 0x%" PRIx64 ")\n", + rss_hf_capa); if (p->hash_proto.proto.ipv6_udp && ((rss_hf_capa & ETH_RSS_NONFRAG_IPV6_UDP) == 0)) - ODP_PRINT("DPDK: hash_proto.ipv6_udp not supported (rss_hf_capa 0x%" PRIx64 ")\n", - rss_hf_capa); + _ODP_PRINT("DPDK: hash_proto.ipv6_udp not supported (rss_hf_capa 0x%" PRIx64 ")\n", + rss_hf_capa); if (p->hash_proto.proto.ipv6_tcp && ((rss_hf_capa & ETH_RSS_NONFRAG_IPV6_TCP) == 0)) - ODP_PRINT("DPDK: hash_proto.ipv6_tcp not supported (rss_hf_capa 0x%" PRIx64 ")\n", - rss_hf_capa); + _ODP_PRINT("DPDK: hash_proto.ipv6_tcp not supported (rss_hf_capa 0x%" PRIx64 ")\n", + rss_hf_capa); hash_proto_to_rss_conf(&pkt_dpdk->rss_conf, &p->hash_proto); @@ -433,7 +431,7 @@ static int dpdk_output_queues_config(pktio_entry_t *pktio_entry, ret = rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info); if (ret) { - ODP_ERR("DPDK: rte_eth_dev_info_get() failed: %d\n", ret); + _ODP_ERR("DPDK: rte_eth_dev_info_get() failed: %d\n", ret); return -1; } @@ -447,12 +445,12 @@ static int dpdk_output_queues_config(pktio_entry_t *pktio_entry, /* Adjust descriptor count */ ret = rte_eth_dev_adjust_nb_rx_tx_desc(pkt_dpdk->port_id, NULL, &num_tx_desc); if (ret && ret != -ENOTSUP) { - ODP_ERR("DPDK: rte_eth_dev_adjust_nb_rx_tx_desc() failed: %d\n", ret); + _ODP_ERR("DPDK: rte_eth_dev_adjust_nb_rx_tx_desc() failed: %d\n", ret); return -1; } pkt_dpdk->num_tx_desc[i] = num_tx_desc; - ODP_DBG("TX queue %" PRIu32 " using %" PRIu16 " descriptors\n", i, num_tx_desc); + _ODP_DBG("TX queue %" PRIu32 " using %" PRIu16 " descriptors\n", i, num_tx_desc); } return 0; } @@ -482,13 +480,13 @@ static int promisc_mode_check(pkt_dpdk_t *pkt_dpdk) ret = rte_eth_promiscuous_enable(pkt_dpdk->port_id); if (ret) { - ODP_DBG("Promisc mode enable not supported: %d\n", ret); + _ODP_DBG("Promisc mode enable not supported: %d\n", ret); return 0; } ret = rte_eth_promiscuous_disable(pkt_dpdk->port_id); if (ret) { - ODP_DBG("Promisc mode disable not supported: %d\n", ret); + _ODP_DBG("Promisc mode disable not supported: %d\n", ret); return 0; } @@ -532,8 +530,7 @@ static int dpdk_init_capability(pktio_entry_t *pktio_entry, if (ret == 0) { capa->set_op.op.mac_addr = 1; } else if (ret != -ENOTSUP && ret != -EPERM) { - ODP_ERR("Failed to set interface default MAC: %d\n", - ret); + _ODP_ERR("Failed to set interface default MAC: %d\n", ret); return -1; } @@ -649,25 +646,25 @@ static int setup_pkt_dpdk(odp_pktio_t pktio ODP_UNUSED, else if (_dpdk_netdev_is_valid(netdev)) pkt_dpdk->port_id = atoi(netdev); else { - ODP_ERR("Invalid interface name!: %s\n", netdev); + _ODP_ERR("Invalid interface name!: %s\n", netdev); return -1; } if (!rte_eth_dev_is_valid_port(pkt_dpdk->port_id)) { - ODP_ERR("Port id=%" PRIu16 " not attached\n", pkt_dpdk->port_id); + _ODP_ERR("Port id=%" PRIu16 " not attached\n", pkt_dpdk->port_id); return -1; } memset(&dev_info, 0, sizeof(struct rte_eth_dev_info)); ret = rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info); if (ret) { - ODP_ERR("Failed to read device info: %d\n", ret); + _ODP_ERR("Failed to read device info: %d\n", ret); return -1; } /* Initialize runtime options */ if (init_options(pktio_entry, &dev_info)) { - ODP_ERR("Initializing runtime options failed\n"); + _ODP_ERR("Initializing runtime options failed\n"); return -1; } @@ -685,7 +682,7 @@ static int setup_pkt_dpdk(odp_pktio_t pktio ODP_UNUSED, mtu = mtu_get_pkt_dpdk(pktio_entry); if (mtu == 0) { - ODP_ERR("Failed to read interface MTU\n"); + _ODP_ERR("Failed to read interface MTU\n"); return -1; } pkt_dpdk->mtu = mtu + _ODP_ETHHDR_LEN; @@ -693,7 +690,7 @@ static int setup_pkt_dpdk(odp_pktio_t pktio ODP_UNUSED, pkt_dpdk->mtu_set = 0; if (dpdk_init_capability(pktio_entry, &dev_info)) { - ODP_ERR("Failed to initialize capability\n"); + _ODP_ERR("Failed to initialize capability\n"); return -1; } @@ -738,8 +735,7 @@ static int dpdk_setup_eth_tx(pktio_entry_t *pktio_entry, rte_eth_dev_socket_id(port_id), &dev_info->default_txconf); if (ret < 0) { - ODP_ERR("Queue setup failed: err=%d, port=%" PRIu8 "\n", - ret, port_id); + _ODP_ERR("Queue setup failed: err=%d, port=%" PRIu8 "\n", ret, port_id); return -1; } } @@ -749,11 +745,11 @@ static int dpdk_setup_eth_tx(pktio_entry_t *pktio_entry, for (i = 0; i < pktio_entry->num_out_queue && i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { ret = rte_eth_dev_set_tx_queue_stats_mapping(port_id, i, i); if (ret) { - ODP_DBG("Mapping per TX queue statistics not supported: %d\n", ret); + _ODP_DBG("Mapping per TX queue statistics not supported: %d\n", ret); break; } } - ODP_DBG("Mapped %" PRIu32 "/%d TX counters\n", i, RTE_ETHDEV_QUEUE_STAT_CNTRS); + _ODP_DBG("Mapped %" PRIu32 "/%d TX counters\n", i, RTE_ETHDEV_QUEUE_STAT_CNTRS); return 0; } @@ -776,19 +772,18 @@ static int dpdk_setup_eth_rx(const pktio_entry_t *pktio_entry, /* Adjust descriptor count */ ret = rte_eth_dev_adjust_nb_rx_tx_desc(port_id, &num_rx_desc, NULL); if (ret && ret != -ENOTSUP) { - ODP_ERR("DPDK: rte_eth_dev_adjust_nb_rx_tx_desc() failed: %d\n", ret); + _ODP_ERR("DPDK: rte_eth_dev_adjust_nb_rx_tx_desc() failed: %d\n", ret); return -1; } - ODP_DBG("RX queues using %" PRIu16 " descriptors\n", num_rx_desc); + _ODP_DBG("RX queues using %" PRIu16 " descriptors\n", num_rx_desc); for (i = 0; i < pktio_entry->num_in_queue; i++) { ret = rte_eth_rx_queue_setup(port_id, i, num_rx_desc, rte_eth_dev_socket_id(port_id), &rxconf, pool->rte_mempool); if (ret < 0) { - ODP_ERR("Queue setup failed: err=%d, port=%" PRIu8 "\n", - ret, port_id); + _ODP_ERR("Queue setup failed: err=%d, port=%" PRIu8 "\n", ret, port_id); return -1; } } @@ -798,11 +793,11 @@ static int dpdk_setup_eth_rx(const pktio_entry_t *pktio_entry, for (i = 0; i < pktio_entry->num_in_queue && i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { ret = rte_eth_dev_set_rx_queue_stats_mapping(port_id, i, i); if (ret) { - ODP_DBG("Mapping per RX queue statistics not supported: %d\n", ret); + _ODP_DBG("Mapping per RX queue statistics not supported: %d\n", ret); break; } } - ODP_DBG("Mapped %" PRIu32 "/%d RX counters\n", i, RTE_ETHDEV_QUEUE_STAT_CNTRS); + _ODP_DBG("Mapped %" PRIu32 "/%d RX counters\n", i, RTE_ETHDEV_QUEUE_STAT_CNTRS); return 0; } @@ -828,7 +823,7 @@ static int dpdk_start(pktio_entry_t *pktio_entry) /* Setup device */ if (dpdk_setup_eth_dev(pktio_entry)) { - ODP_ERR("Failed to configure device\n"); + _ODP_ERR("Failed to configure device\n"); return -1; } @@ -844,8 +839,8 @@ static int dpdk_start(pktio_entry_t *pktio_entry) if (pkt_dpdk->mtu_set && pktio_entry->capa.set_op.op.maxlen) { ret = dpdk_maxlen_set(pktio_entry, pkt_dpdk->mtu, 0); if (ret) { - ODP_ERR("Restoring device MTU failed: err=%d, port=%" PRIu8 "\n", - ret, port_id); + _ODP_ERR("Restoring device MTU failed: err=%d, port=%" PRIu8 "\n", + ret, port_id); return -1; } } @@ -853,8 +848,7 @@ static int dpdk_start(pktio_entry_t *pktio_entry) /* Start device */ ret = rte_eth_dev_start(port_id); if (ret < 0) { - ODP_ERR("Device start failed: err=%d, port=%" PRIu8 "\n", - ret, port_id); + _ODP_ERR("Device start failed: err=%d, port=%" PRIu8 "\n", ret, port_id); return -1; } @@ -1002,8 +996,8 @@ static int recv_pkt_dpdk(pktio_entry_t *pktio_entry, int index, odp_packet_t min_burst[min]; uint16_t i; - ODP_DBG("PMD requires >%d buffers burst. Current %d, dropped " - "%d\n", min, num, min - num); + _ODP_DBG("PMD requires >%d buffers burst. Current %d, dropped %d\n", + min, num, min - num); nb_rx = rte_eth_rx_burst(port_id, (uint16_t)index, (struct rte_mbuf **)min_burst, min); @@ -1197,7 +1191,7 @@ static uint32_t _dpdk_vdev_mtu(uint16_t port_id) ret = ioctl(sockfd, SIOCGIFMTU, &ifr); close(sockfd); if (ret < 0) { - ODP_DBG("ioctl SIOCGIFMTU error\n"); + _ODP_DBG("ioctl SIOCGIFMTU error\n"); return 0; } @@ -1239,7 +1233,7 @@ static int promisc_mode_set_pkt_dpdk(pktio_entry_t *pktio_entry, int enable) ret = rte_eth_promiscuous_disable(port_id); if (ret) { - ODP_ERR("Setting promisc mode failed: %d\n", ret); + _ODP_ERR("Setting promisc mode failed: %d\n", ret); return -1; } return 0; @@ -1252,7 +1246,7 @@ static int promisc_mode_get_pkt_dpdk(pktio_entry_t *pktio_entry) ret = rte_eth_promiscuous_get(port_id); if (ret < 0) { - ODP_ERR("Getting promisc mode failed: %d\n", ret); + _ODP_ERR("Getting promisc mode failed: %d\n", ret); return -1; } return ret; @@ -1288,9 +1282,9 @@ static int link_status_pkt_dpdk(pktio_entry_t *pktio_entry) ret = rte_eth_link_get_nowait(pkt_priv(pktio_entry)->port_id, &link); if (ret) { if (ret == -ENOTSUP) - ODP_DBG("rte_eth_link_get_nowait() not supported\n"); + _ODP_DBG("rte_eth_link_get_nowait() not supported\n"); else - ODP_ERR("rte_eth_link_get_nowait() failed\n"); + _ODP_ERR("rte_eth_link_get_nowait() failed\n"); return ODP_PKTIO_LINK_STATUS_UNKNOWN; } @@ -1314,10 +1308,10 @@ static int dpdk_link_info(pktio_entry_t *pktio_entry, odp_pktio_link_info_t *inf ret = rte_eth_dev_flow_ctrl_get(port_id, &fc_conf); if (ret) { if (ret != -ENOTSUP) { - ODP_ERR("rte_eth_dev_flow_ctrl_get() failed\n"); + _ODP_ERR("rte_eth_dev_flow_ctrl_get() failed\n"); return -1; } - ODP_DBG("rte_eth_dev_flow_ctrl_get() not supported\n"); + _ODP_DBG("rte_eth_dev_flow_ctrl_get() not supported\n"); link_info.pause_rx = ODP_PKTIO_LINK_PAUSE_UNKNOWN; link_info.pause_tx = ODP_PKTIO_LINK_PAUSE_UNKNOWN; } else { @@ -1336,10 +1330,10 @@ static int dpdk_link_info(pktio_entry_t *pktio_entry, odp_pktio_link_info_t *inf ret = rte_eth_link_get_nowait(port_id, &link); if (ret) { if (ret != -ENOTSUP) { - ODP_ERR("rte_eth_link_get_nowait() failed\n"); + _ODP_ERR("rte_eth_link_get_nowait() failed\n"); return -1; } - ODP_DBG("rte_eth_link_get_nowait() not supported\n"); + _ODP_DBG("rte_eth_link_get_nowait() not supported\n"); link_info.autoneg = ODP_PKTIO_LINK_AUTONEG_UNKNOWN; link_info.duplex = ODP_PKTIO_LINK_DUPLEX_UNKNOWN; link_info.speed = ODP_PKTIO_LINK_SPEED_UNKNOWN; @@ -1426,7 +1420,7 @@ static int dpdk_extra_stat_info(pktio_entry_t *pktio_entry, num_stats = rte_eth_xstats_get_names(port_id, NULL, 0); if (num_stats < 0) { - ODP_ERR("rte_eth_xstats_get_names() failed: %d\n", num_stats); + _ODP_ERR("rte_eth_xstats_get_names() failed: %d\n", num_stats); return num_stats; } else if (info == NULL || num == 0 || num_stats == 0) { return num_stats; @@ -1436,7 +1430,7 @@ static int dpdk_extra_stat_info(pktio_entry_t *pktio_entry, ret = rte_eth_xstats_get_names(port_id, xstats_names, num_stats); if (ret < 0 || ret > num_stats) { - ODP_ERR("rte_eth_xstats_get_names() failed: %d\n", ret); + _ODP_ERR("rte_eth_xstats_get_names() failed: %d\n", ret); return -1; } num_stats = ret; @@ -1456,7 +1450,7 @@ static int dpdk_extra_stats(pktio_entry_t *pktio_entry, num_stats = rte_eth_xstats_get(port_id, NULL, 0); if (num_stats < 0) { - ODP_ERR("rte_eth_xstats_get() failed: %d\n", num_stats); + _ODP_ERR("rte_eth_xstats_get() failed: %d\n", num_stats); return num_stats; } else if (stats == NULL || num == 0 || num_stats == 0) { return num_stats; @@ -1466,7 +1460,7 @@ static int dpdk_extra_stats(pktio_entry_t *pktio_entry, ret = rte_eth_xstats_get(port_id, xstats, num_stats); if (ret < 0 || ret > num_stats) { - ODP_ERR("rte_eth_xstats_get() failed: %d\n", ret); + _ODP_ERR("rte_eth_xstats_get() failed: %d\n", ret); return -1; } num_stats = ret; @@ -1486,7 +1480,7 @@ static int dpdk_extra_stat_counter(pktio_entry_t *pktio_entry, uint32_t id, ret = rte_eth_xstats_get_by_id(port_id, &xstat_id, stat, 1); if (ret != 1) { - ODP_ERR("rte_eth_xstats_get_by_id() failed: %d\n", ret); + _ODP_ERR("rte_eth_xstats_get_by_id() failed: %d\n", ret); return -1; } @@ -1500,14 +1494,14 @@ static int dpdk_pktin_stats(pktio_entry_t *pktio_entry, uint32_t index, int ret; if (odp_unlikely(index > RTE_ETHDEV_QUEUE_STAT_CNTRS - 1)) { - ODP_ERR("DPDK supports max %d per queue counters\n", - RTE_ETHDEV_QUEUE_STAT_CNTRS); + _ODP_ERR("DPDK supports max %d per queue counters\n", + RTE_ETHDEV_QUEUE_STAT_CNTRS); return -1; } ret = rte_eth_stats_get(pkt_priv(pktio_entry)->port_id, &rte_stats); if (odp_unlikely(ret)) { - ODP_ERR("Failed to read DPDK pktio stats: %d\n", ret); + _ODP_ERR("Failed to read DPDK pktio stats: %d\n", ret); return -1; } @@ -1527,14 +1521,14 @@ static int dpdk_pktout_stats(pktio_entry_t *pktio_entry, uint32_t index, int ret; if (odp_unlikely(index > RTE_ETHDEV_QUEUE_STAT_CNTRS - 1)) { - ODP_ERR("DPDK supports max %d per queue counters\n", - RTE_ETHDEV_QUEUE_STAT_CNTRS); + _ODP_ERR("DPDK supports max %d per queue counters\n", + RTE_ETHDEV_QUEUE_STAT_CNTRS); return -1; } ret = rte_eth_stats_get(pkt_priv(pktio_entry)->port_id, &rte_stats); if (odp_unlikely(ret)) { - ODP_ERR("Failed to read DPDK pktio stats: %d\n", ret); + _ODP_ERR("Failed to read DPDK pktio stats: %d\n", ret); return -1; } diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index e400540d9..b0df7997a 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -126,23 +126,23 @@ static int read_config_file(pool_global_t *pool_gbl) const char *str; int val = 0; - ODP_PRINT("Pool config:\n"); + _ODP_PRINT("Pool config:\n"); str = "pool.pkt.max_num"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } if (val < 0 || val > CONFIG_POOL_MAX_NUM) { - ODP_ERR("Bad value %s = %u\n", str, val); + _ODP_ERR("Bad value %s = %u\n", str, val); return -1; } pool_gbl->config.pkt_max_num = val; - ODP_PRINT(" %s: %i\n", str, val); + _ODP_PRINT(" %s: %i\n", str, val); - ODP_PRINT("\n"); + _ODP_PRINT("\n"); return 0; } @@ -176,14 +176,14 @@ int _odp_pool_init_global(void) pool->pool_idx = i; } - ODP_DBG("\nPool init global\n"); - ODP_DBG(" event_hdr_t size: %zu\n", sizeof(_odp_event_hdr_t)); - ODP_DBG(" odp_buffer_hdr_t size: %zu\n", sizeof(odp_buffer_hdr_t)); - ODP_DBG(" odp_packet_hdr_t size: %zu\n", sizeof(odp_packet_hdr_t)); - ODP_DBG(" odp_timeout_hdr_t size: %zu\n", sizeof(odp_timeout_hdr_t)); - ODP_DBG(" odp_event_vector_hdr_t size: %zu\n", sizeof(odp_event_vector_hdr_t)); + _ODP_DBG("\nPool init global\n"); + _ODP_DBG(" event_hdr_t size: %zu\n", sizeof(_odp_event_hdr_t)); + _ODP_DBG(" odp_buffer_hdr_t size: %zu\n", sizeof(odp_buffer_hdr_t)); + _ODP_DBG(" odp_packet_hdr_t size: %zu\n", sizeof(odp_packet_hdr_t)); + _ODP_DBG(" odp_timeout_hdr_t size: %zu\n", sizeof(odp_timeout_hdr_t)); + _ODP_DBG(" odp_event_vector_hdr_t size: %zu\n", sizeof(odp_event_vector_hdr_t)); - ODP_DBG("\n"); + _ODP_DBG("\n"); return 0; } @@ -202,7 +202,7 @@ int _odp_pool_term_global(void) ret = odp_shm_free(_odp_pool_glb->shm); if (ret < 0) - ODP_ERR("SHM free failed\n"); + _ODP_ERR("SHM free failed\n"); return ret; } @@ -397,7 +397,7 @@ odp_dpdk_mbuf_ctor(struct rte_mempool *mp, #define CHECK_U16_OVERFLOW(X) do { \ if (odp_unlikely(X > UINT16_MAX)) { \ - ODP_ERR("Invalid size: %d\n", X); \ + _ODP_ERR("Invalid size: %d\n", X); \ UNLOCK(&pool->lock); \ return ODP_POOL_INVALID; \ } \ @@ -413,43 +413,42 @@ static int check_params(const odp_pool_param_t *params) switch (params->type) { case ODP_POOL_BUFFER: if (params->buf.num == 0) { - ODP_ERR("buf.num zero\n"); + _ODP_ERR("buf.num zero\n"); return -1; } if (params->buf.num > capa.buf.max_num) { - ODP_ERR("buf.num too large %u\n", params->buf.num); + _ODP_ERR("buf.num too large %u\n", params->buf.num); return -1; } if (params->buf.size > capa.buf.max_size) { - ODP_ERR("buf.size too large %u\n", params->buf.size); + _ODP_ERR("buf.size too large %u\n", params->buf.size); return -1; } if (params->buf.align > capa.buf.max_align) { - ODP_ERR("buf.align too large %u\n", params->buf.align); + _ODP_ERR("buf.align too large %u\n", params->buf.align); return -1; } if (!_ODP_CHECK_IS_POWER2(params->buf.align)) { - ODP_ERR("buf.align not power of two %u\n", params->buf.align); + _ODP_ERR("buf.align not power of two %u\n", params->buf.align); return -1; } if (params->buf.cache_size > capa.buf.max_cache_size) { - ODP_ERR("buf.cache_size too large %u\n", - params->buf.cache_size); + _ODP_ERR("buf.cache_size too large %u\n", params->buf.cache_size); return -1; } if (params->buf.uarea_size > capa.buf.max_uarea_size) { - ODP_ERR("buf.uarea_size too large %u\n", params->buf.uarea_size); + _ODP_ERR("buf.uarea_size too large %u\n", params->buf.uarea_size); return -1; } if (params->stats.all & ~capa.buf.stats.all) { - ODP_ERR("Unsupported pool statistics counter\n"); + _ODP_ERR("Unsupported pool statistics counter\n"); return -1; } @@ -457,57 +456,52 @@ static int check_params(const odp_pool_param_t *params) case ODP_POOL_PACKET: if (params->pkt.align > capa.pkt.max_align) { - ODP_ERR("pkt.align too large %u\n", params->pkt.align); + _ODP_ERR("pkt.align too large %u\n", params->pkt.align); return -1; } if (!_ODP_CHECK_IS_POWER2(params->pkt.align)) { - ODP_ERR("pkt.align not power of two %u\n", params->pkt.align); + _ODP_ERR("pkt.align not power of two %u\n", params->pkt.align); return -1; } if (params->pkt.num == 0) { - ODP_ERR("pkt.num zero\n"); + _ODP_ERR("pkt.num zero\n"); return -1; } if (params->pkt.num > capa.pkt.max_num) { - ODP_ERR("pkt.num too large %u\n", params->pkt.num); + _ODP_ERR("pkt.num too large %u\n", params->pkt.num); return -1; } if (params->pkt.max_num > capa.pkt.max_num) { - ODP_ERR("pkt.max_num too large %u\n", - params->pkt.max_num); + _ODP_ERR("pkt.max_num too large %u\n", params->pkt.max_num); return -1; } if (params->pkt.seg_len > capa.pkt.max_seg_len) { - ODP_ERR("pkt.seg_len too large %u\n", - params->pkt.seg_len); + _ODP_ERR("pkt.seg_len too large %u\n", params->pkt.seg_len); return -1; } if (params->pkt.uarea_size > capa.pkt.max_uarea_size) { - ODP_ERR("pkt.uarea_size too large %u\n", - params->pkt.uarea_size); + _ODP_ERR("pkt.uarea_size too large %u\n", params->pkt.uarea_size); return -1; } if (params->pkt.headroom > capa.pkt.max_headroom) { - ODP_ERR("pkt.headroom too large %u\n", - params->pkt.headroom); + _ODP_ERR("pkt.headroom too large %u\n", params->pkt.headroom); return -1; } if (params->pkt.cache_size > capa.pkt.max_cache_size) { - ODP_ERR("pkt.cache_size too large %u\n", - params->pkt.cache_size); + _ODP_ERR("pkt.cache_size too large %u\n", params->pkt.cache_size); return -1; } if (params->stats.all & ~capa.pkt.stats.all) { - ODP_ERR("Unsupported pool statistics counter\n"); + _ODP_ERR("Unsupported pool statistics counter\n"); return -1; } @@ -515,28 +509,27 @@ static int check_params(const odp_pool_param_t *params) case ODP_POOL_TIMEOUT: if (params->tmo.num == 0) { - ODP_ERR("tmo.num zero\n"); + _ODP_ERR("tmo.num zero\n"); return -1; } if (params->tmo.num > capa.tmo.max_num) { - ODP_ERR("tmo.num too large %u\n", params->tmo.num); + _ODP_ERR("tmo.num too large %u\n", params->tmo.num); return -1; } if (params->tmo.cache_size > capa.tmo.max_cache_size) { - ODP_ERR("tmo.cache_size too large %u\n", - params->tmo.cache_size); + _ODP_ERR("tmo.cache_size too large %u\n", params->tmo.cache_size); return -1; } if (params->tmo.uarea_size > capa.tmo.max_uarea_size) { - ODP_ERR("tmo.uarea_size too large %u\n", params->tmo.uarea_size); + _ODP_ERR("tmo.uarea_size too large %u\n", params->tmo.uarea_size); return -1; } if (params->stats.all & ~capa.tmo.stats.all) { - ODP_ERR("Unsupported pool statistics counter\n"); + _ODP_ERR("Unsupported pool statistics counter\n"); return -1; } @@ -544,44 +537,44 @@ static int check_params(const odp_pool_param_t *params) case ODP_POOL_VECTOR: if (params->vector.num == 0) { - ODP_ERR("vector.num zero\n"); + _ODP_ERR("vector.num zero\n"); return -1; } if (params->vector.num > capa.vector.max_num) { - ODP_ERR("vector.num too large %u\n", params->vector.num); + _ODP_ERR("vector.num too large %u\n", params->vector.num); return -1; } if (params->vector.max_size == 0) { - ODP_ERR("vector.max_size zero\n"); + _ODP_ERR("vector.max_size zero\n"); return -1; } if (params->vector.max_size > capa.vector.max_size) { - ODP_ERR("vector.max_size too large %u\n", params->vector.max_size); + _ODP_ERR("vector.max_size too large %u\n", params->vector.max_size); return -1; } if (params->vector.cache_size > capa.vector.max_cache_size) { - ODP_ERR("vector.cache_size too large %u\n", params->vector.cache_size); + _ODP_ERR("vector.cache_size too large %u\n", params->vector.cache_size); return -1; } if (params->vector.uarea_size > capa.vector.max_uarea_size) { - ODP_ERR("vector.uarea_size too large %u\n", params->vector.uarea_size); + _ODP_ERR("vector.uarea_size too large %u\n", params->vector.uarea_size); return -1; } if (params->stats.all & ~capa.vector.stats.all) { - ODP_ERR("Unsupported pool statistics counter\n"); + _ODP_ERR("Unsupported pool statistics counter\n"); return -1; } break; default: - ODP_ERR("bad pool type %i\n", params->type); + _ODP_ERR("bad pool type %i\n", params->type); return -1; } @@ -608,22 +601,22 @@ static unsigned int calc_cache_size(uint32_t pool_size, uint32_t max_num) if (odp_unlikely(cache_size == 0)) { cache_size = RTE_MIN(max_num, max_supported); - ODP_DBG("Using nonoptimal cache size: %d\n", cache_size); + _ODP_DBG("Using nonoptimal cache size: %d\n", cache_size); } /* Cache size of one exposes DPDK implementation bug */ if (cache_size == 1) cache_size = 0; - ODP_DBG("Cache_size: %d\n", cache_size); + _ODP_DBG("Cache_size: %d\n", cache_size); if (num_threads && cache_size) { unsigned int total_cache_size = num_threads * cache_size; if (total_cache_size >= pool_size) - ODP_DBG("Entire pool fits into thread local caches. " - "Pool starvation may occur if the pool is used " - "by multiple threads.\n"); + _ODP_DBG("Entire pool fits into thread local caches. " + "Pool starvation may occur if the pool is used " + "by multiple threads.\n"); } return cache_size; @@ -735,8 +728,8 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, num = params->buf.num; event_type = ODP_EVENT_BUFFER; - ODP_DBG("type: buffer, name: %s, num: %u, size: %u, align: %u\n", - pool_name, num, params->buf.size, params->buf.align); + _ODP_DBG("type: buffer, name: %s, num: %u, size: %u, align: %u\n", + pool_name, num, params->buf.size, params->buf.align); break; case ODP_POOL_PACKET: headroom = RTE_PKTMBUF_HEADROOM; @@ -769,8 +762,8 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, /* Round down the block size to 16 bits */ if (blk_size > UINT16_MAX) { blk_size = UINT16_MAX; - ODP_DBG("Packet pool block size rounded down to %" PRIu32 "\n", - blk_size); + _ODP_DBG("Packet pool block size rounded down to %" PRIu32 "\n", + blk_size); } hdr_size = sizeof(odp_packet_hdr_t); @@ -780,9 +773,9 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, event_type = ODP_EVENT_PACKET; uarea_size = params->pkt.uarea_size; - ODP_DBG("type: packet, name: %s, num: %u, len: %u, blk_size: %u, " - "uarea_size: %d, hdr_size: %zu\n", pool_name, num, params->pkt.len, - blk_size, uarea_size, hdr_size); + _ODP_DBG("type: packet, name: %s, num: %u, len: %u, blk_size: %u, " + "uarea_size: %d, hdr_size: %zu\n", pool_name, num, params->pkt.len, + blk_size, uarea_size, hdr_size); break; case ODP_POOL_TIMEOUT: hdr_size = sizeof(odp_timeout_hdr_t); @@ -792,7 +785,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, uarea_size = params->tmo.uarea_size; event_type = ODP_EVENT_TIMEOUT; - ODP_DBG("type: tmo, name: %s, num: %u\n", pool_name, num); + _ODP_DBG("type: tmo, name: %s, num: %u\n", pool_name, num); break; case ODP_POOL_VECTOR: hdr_size = sizeof(odp_event_vector_hdr_t) + @@ -803,10 +796,10 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, uarea_size = params->vector.uarea_size; event_type = ODP_EVENT_PACKET_VECTOR; - ODP_DBG("type: vector, name: %s, num: %u\n", pool_name, num); + _ODP_DBG("type: vector, name: %s, num: %u\n", pool_name, num); break; default: - ODP_ERR("Bad pool type %i\n", (int)type); + _ODP_ERR("Bad pool type %i\n", (int)type); UNLOCK(&pool->lock); return ODP_POOL_INVALID; } @@ -820,8 +813,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, mbp_ctor_arg.mbuf_priv_size = mb_ctor_arg.seg_buf_offset - sizeof(struct rte_mbuf); - ODP_DBG("Metadata size: %u, mb_size %d\n", - mb_ctor_arg.seg_buf_offset, mb_size); + _ODP_DBG("Metadata size: %u, mb_size %d\n", mb_ctor_arg.seg_buf_offset, mb_size); cache_size = calc_cache_size(num, cache_size); @@ -847,8 +839,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, rte_socket_id(), 0); } if (mp == NULL) { - ODP_ERR("Cannot init DPDK mbuf pool: %s\n", - rte_strerror(rte_errno)); + _ODP_ERR("Cannot init DPDK mbuf pool: %s\n", rte_strerror(rte_errno)); UNLOCK(&pool->lock); return ODP_POOL_INVALID; } @@ -866,7 +857,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, pool->params = *params; if (reserve_uarea(pool, uarea_size, num)) { - ODP_ERR("User area SHM reserve failed\n"); + _ODP_ERR("User area SHM reserve failed\n"); rte_mempool_free(mp); UNLOCK(&pool->lock); return ODP_POOL_INVALID; @@ -876,11 +867,10 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, rte_mempool_obj_iter(mp, odp_dpdk_mbuf_ctor, &mb_ctor_arg); pool->rte_mempool = mp; - ODP_DBG("Header/element/trailer size: %u/%u/%u, " - "total pool size: %lu\n", - mp->header_size, mp->elt_size, mp->trailer_size, - (unsigned long)((mp->header_size + mp->elt_size + - mp->trailer_size) * num)); + _ODP_DBG("Header/element/trailer size: %u/%u/%u, total pool size: %lu\n", + mp->header_size, mp->elt_size, mp->trailer_size, + (unsigned long)((mp->header_size + mp->elt_size + + mp->trailer_size) * num)); UNLOCK(&pool->lock); pool_hdl = _odp_pool_handle(pool); break; @@ -922,11 +912,11 @@ odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl) odp_event_t event; pool_t *pool; - ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); + _ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); pool = _odp_pool_entry(pool_hdl); - ODP_ASSERT(pool->type == ODP_POOL_BUFFER); + _ODP_ASSERT(pool->type == ODP_POOL_BUFFER); event = _odp_event_alloc(pool); if (odp_likely(event != ODP_EVENT_INVALID)) @@ -939,11 +929,11 @@ int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num) { pool_t *pool; - ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); + _ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); pool = _odp_pool_entry(pool_hdl); - ODP_ASSERT(pool->type == ODP_POOL_BUFFER); + _ODP_ASSERT(pool->type == ODP_POOL_BUFFER); return _odp_event_alloc_multi(pool, (_odp_event_hdr_t **)buf, num); } @@ -965,9 +955,9 @@ void odp_pool_print_all(void) const char *name; char type_c; - ODP_PRINT("\nList of all pools\n"); - ODP_PRINT("-----------------\n"); - ODP_PRINT(" idx %-*s type free tot cache elt_len ext\n", col_width, "name"); + _ODP_PRINT("\nList of all pools\n"); + _ODP_PRINT("-----------------\n"); + _ODP_PRINT(" idx %-*s type free tot cache elt_len ext\n", col_width, "name"); for (i = 0; i < ODP_CONFIG_POOLS; i++) { pool_t *pool = _odp_pool_entry_from_idx(i); @@ -998,11 +988,11 @@ void odp_pool_print_all(void) (type == ODP_POOL_TIMEOUT) ? 'T' : (type == ODP_POOL_VECTOR) ? 'V' : '-'; - ODP_PRINT("%4u %-*s %c %6" PRIu64 " %6" PRIu32 " %6" PRIu32 " %8" PRIu32 " " - "%" PRIu8 "\n", index, col_width, name, type_c, available, tot, - cache_size, elt_len, ext); + _ODP_PRINT("%4u %-*s %c %6" PRIu64 " %6" PRIu32 " %6" PRIu32 " %8" PRIu32 " " + "%" PRIu8 "\n", index, col_width, name, type_c, available, tot, + cache_size, elt_len, ext); } - ODP_PRINT("\n"); + _ODP_PRINT("\n"); } static void mempool_addr_range(struct rte_mempool *mp ODP_UNUSED, void *opaque, @@ -1060,13 +1050,13 @@ int odp_pool_destroy(odp_pool_t pool_hdl) pool_t *pool; if (pool_hdl == ODP_POOL_INVALID) { - ODP_ERR("Invalid pool handle\n"); + _ODP_ERR("Invalid pool handle\n"); return -1; } pool = _odp_pool_entry(pool_hdl); if (pool->rte_mempool == NULL) { - ODP_ERR("No rte_mempool handle available\n"); + _ODP_ERR("No rte_mempool handle available\n"); return -1; } @@ -1105,11 +1095,11 @@ int odp_pool_stats(odp_pool_t pool_hdl, odp_pool_stats_t *stats) uint16_t first, last; if (odp_unlikely(pool_hdl == ODP_POOL_INVALID)) { - ODP_ERR("Invalid pool handle\n"); + _ODP_ERR("Invalid pool handle\n"); return -1; } if (odp_unlikely(stats == NULL)) { - ODP_ERR("Output buffer NULL\n"); + _ODP_ERR("Output buffer NULL\n"); return -1; } @@ -1201,49 +1191,47 @@ static int check_pool_ext_param(const odp_pool_ext_param_t *param) param->pkt.app_header_size; if (param->type != ODP_POOL_PACKET) { - ODP_ERR("Pool type not supported\n"); + _ODP_ERR("Pool type not supported\n"); return -1; } if (odp_pool_ext_capability(param->type, &capa)) { - ODP_ERR("Capa failed\n"); + _ODP_ERR("Capa failed\n"); return -1; } if (param->cache_size > capa.max_cache_size) { - ODP_ERR("Too large cache size %u\n", param->cache_size); + _ODP_ERR("Too large cache size %u\n", param->cache_size); return -1; } if (param->stats.all != capa.stats.all) { - ODP_ERR("Pool statistics not supported\n"); + _ODP_ERR("Pool statistics not supported\n"); return -1; } if (param->pkt.num_buf > capa.pkt.max_num_buf) { - ODP_ERR("Too many packet buffers\n"); + _ODP_ERR("Too many packet buffers\n"); return -1; } if (param->pkt.buf_size > capa.pkt.max_buf_size) { - ODP_ERR("Too large packet buffer size %u\n", - param->pkt.buf_size); + _ODP_ERR("Too large packet buffer size %u\n", param->pkt.buf_size); return -1; } if (param->pkt.uarea_size > capa.pkt.max_uarea_size) { - ODP_ERR("Too large user area size %u\n", param->pkt.uarea_size); + _ODP_ERR("Too large user area size %u\n", param->pkt.uarea_size); return -1; } if (param->pkt.headroom > capa.pkt.max_headroom) { - ODP_ERR("Too large headroom size\n"); + _ODP_ERR("Too large headroom size\n"); return -1; } if (head_offset % capa.pkt.min_head_align) { - ODP_ERR("Head pointer not %u byte aligned\n", - capa.pkt.min_head_align); + _ODP_ERR("Head pointer not %u byte aligned\n", capa.pkt.min_head_align); return -1; } @@ -1293,10 +1281,10 @@ odp_pool_t odp_pool_ext_create(const char *name, blk_size = buf_size - SIZEOF_OBJHDR - hdr_size; num = params->pkt.num_buf; - ODP_DBG("type: packet, name: %s, num: %u, len: %u, blk_size: %u, " - "uarea_size: %d, hdr_size: %zu\n", - pool_name, num, buf_size, blk_size, - params->pkt.uarea_size, hdr_size); + _ODP_DBG("type: packet, name: %s, num: %u, len: %u, blk_size: %u, " + "uarea_size: %d, hdr_size: %zu\n", + pool_name, num, buf_size, blk_size, + params->pkt.uarea_size, hdr_size); cache_size = params->cache_size; cache_size = calc_cache_size(num, cache_size); @@ -1309,13 +1297,13 @@ odp_pool_t odp_pool_ext_create(const char *name, rte_socket_id(), 0); if (mp == NULL) { - ODP_ERR("Failed to create empty DPDK packet pool\n"); + _ODP_ERR("Failed to create empty DPDK packet pool\n"); goto error; } if (rte_mempool_set_ops_byname(mp, rte_mbuf_best_mempool_ops(), NULL)) { - ODP_ERR("Failed setting mempool operations\n"); + _ODP_ERR("Failed setting mempool operations\n"); goto error; } @@ -1332,7 +1320,7 @@ odp_pool_t odp_pool_ext_create(const char *name, * Fedora. */ if (rte_mempool_get_ops(mp->ops_index)->alloc(mp)) { - ODP_ERR("Mempool alloc operation failed\n"); + _ODP_ERR("Mempool alloc operation failed\n"); goto error; } @@ -1348,17 +1336,15 @@ odp_pool_t odp_pool_ext_create(const char *name, strcpy(pool->name, pool_name); if (reserve_uarea(pool, params->pkt.uarea_size, num)) { - ODP_ERR("User area SHM reserve failed\n"); + _ODP_ERR("User area SHM reserve failed\n"); goto error; } pool->rte_mempool = mp; - ODP_DBG("Header/element/trailer size: %u/%u/%u, " - "total pool size: %lu\n", - mp->header_size, mp->elt_size, mp->trailer_size, - (unsigned long)((mp->header_size + mp->elt_size + - mp->trailer_size) * - num)); + _ODP_DBG("Header/element/trailer size: %u/%u/%u, total pool size: %lu\n", + mp->header_size, mp->elt_size, mp->trailer_size, + (unsigned long)((mp->header_size + mp->elt_size + + mp->trailer_size) * num)); UNLOCK(&pool->lock); pool_hdl = _odp_pool_handle(pool); break; @@ -1378,37 +1364,37 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, uint32_t num_populated; if (pool_hdl == ODP_POOL_INVALID) { - ODP_ERR("Bad pool handle\n"); + _ODP_ERR("Bad pool handle\n"); return -1; } pool = _odp_pool_entry(pool_hdl); if (pool->type != ODP_POOL_PACKET || pool->pool_ext == 0) { - ODP_ERR("Bad pool type\n"); + _ODP_ERR("Bad pool type\n"); return -1; } if (buf_size != pool->ext_param.pkt.buf_size) { - ODP_ERR("Bad buffer size\n"); + _ODP_ERR("Bad buffer size\n"); return -1; } num_populated = pool->num_populated; if (num_populated + num > pool->num) { - ODP_ERR("Trying to over populate the pool\n"); + _ODP_ERR("Trying to over populate the pool\n"); return -1; } if ((num_populated + num == pool->num) && !(flags & ODP_POOL_POPULATE_DONE)) { - ODP_ERR("Missing ODP_POOL_POPULATE_DONE flag\n"); + _ODP_ERR("Missing ODP_POOL_POPULATE_DONE flag\n"); return -1; } if ((num_populated + num < pool->num) && flags) { - ODP_ERR("Unexpected flags: 0x%x\n", flags); + _ODP_ERR("Unexpected flags: 0x%x\n", flags); return -1; } @@ -1427,13 +1413,13 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, * requirement also for buffers. */ if ((uintptr_t)buf[i] & (ODP_CACHE_LINE_SIZE - 1)) { - ODP_ERR("Buffer address (%p) does not meet alignment requirements\n", - buf[i]); + _ODP_ERR("Buffer address (%p) does not meet alignment requirements\n", + buf[i]); return -1; } if (rte_mempool_ops_enqueue_bulk(mp, (void *const *)&mb, 1) < 0) { - ODP_ERR("Failed to enqueue buffer to rte_mempool\n"); + _ODP_ERR("Failed to enqueue buffer to rte_mempool\n"); return -1; } @@ -1444,7 +1430,7 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, memhdr = rte_zmalloc(NULL, sizeof(*memhdr), 0); if (!memhdr) { - ODP_ERR("Failed to allocate rte_mempool_memhdr\n"); + _ODP_ERR("Failed to allocate rte_mempool_memhdr\n"); return -1; } diff --git a/platform/linux-dpdk/odp_queue_basic.c b/platform/linux-dpdk/odp_queue_basic.c index 401ce79a4..a83164e2f 100644 --- a/platform/linux-dpdk/odp_queue_basic.c +++ b/platform/linux-dpdk/odp_queue_basic.c @@ -67,27 +67,27 @@ static int read_config_file(queue_global_t *_odp_queue_glb) uint32_t val_u32; int val = 0; - ODP_PRINT("Queue config:\n"); + _ODP_PRINT("Queue config:\n"); str = "queue_basic.max_queue_size"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } val_u32 = val; if (val_u32 > MAX_QUEUE_SIZE || val_u32 < MIN_QUEUE_SIZE) { - ODP_ERR("Bad value %s = %u\n", str, val_u32); + _ODP_ERR("Bad value %s = %u\n", str, val_u32); return -1; } _odp_queue_glb->config.max_queue_size = val_u32; - ODP_PRINT(" %s: %u\n", str, val_u32); + _ODP_PRINT(" %s: %u\n", str, val_u32); str = "queue_basic.default_queue_size"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } @@ -95,12 +95,12 @@ static int read_config_file(queue_global_t *_odp_queue_glb) if (val_u32 > _odp_queue_glb->config.max_queue_size || val_u32 < MIN_QUEUE_SIZE) { - ODP_ERR("Bad value %s = %u\n", str, val_u32); + _ODP_ERR("Bad value %s = %u\n", str, val_u32); return -1; } _odp_queue_glb->config.default_queue_size = val_u32; - ODP_PRINT(" %s: %u\n\n", str, val_u32); + _ODP_PRINT(" %s: %u\n\n", str, val_u32); return 0; } @@ -113,7 +113,7 @@ static int queue_init_global(void) queue_lf_func_t *lf_func; odp_queue_capability_t capa; - ODP_DBG("Starts...\n"); + _ODP_DBG("Starts...\n"); /* Fill in queue entry field offsets for inline functions */ memset(&_odp_queue_inline_offset, 0, @@ -155,12 +155,12 @@ static int queue_init_global(void) queue_capa(&capa, 0); - ODP_DBG("... done.\n"); - ODP_DBG(" queue_entry_t size %zu\n", sizeof(queue_entry_t)); - ODP_DBG(" max num queues %u\n", capa.max_queues); - ODP_DBG(" max queue size %u\n", capa.plain.max_size); - ODP_DBG(" max num lockfree %u\n", capa.plain.lockfree.max_num); - ODP_DBG(" max lockfree size %u\n\n", capa.plain.lockfree.max_size); + _ODP_DBG("... done.\n"); + _ODP_DBG(" queue_entry_t size %zu\n", sizeof(queue_entry_t)); + _ODP_DBG(" max num queues %u\n", capa.max_queues); + _ODP_DBG(" max queue size %u\n", capa.plain.max_size); + _ODP_DBG(" max num lockfree %u\n", capa.plain.lockfree.max_num); + _ODP_DBG(" max lockfree size %u\n\n", capa.plain.lockfree.max_size); return 0; } @@ -185,7 +185,7 @@ static int queue_term_global(void) queue = qentry_from_index(i); LOCK(queue); if (queue->status != QUEUE_STATUS_FREE) { - ODP_ERR("Not destroyed queue: %s\n", queue->name); + _ODP_ERR("Not destroyed queue: %s\n", queue->name); ret = -1; } UNLOCK(queue); @@ -194,7 +194,7 @@ static int queue_term_global(void) _odp_queue_lf_term_global(); if (odp_shm_free(_odp_queue_glb->queue_gbl_shm)) { - ODP_ERR("shm free failed"); + _ODP_ERR("shm free failed"); ret = -1; } @@ -255,7 +255,7 @@ static odp_queue_t queue_create(const char *name, if (type == ODP_QUEUE_TYPE_SCHED) { if (param->sched.prio < odp_schedule_min_prio() || param->sched.prio > odp_schedule_max_prio()) { - ODP_ERR("Bad queue priority: %i\n", param->sched.prio); + _ODP_ERR("Bad queue priority: %i\n", param->sched.prio); return ODP_QUEUE_INVALID; } } @@ -338,7 +338,7 @@ static odp_queue_t queue_create(const char *name, if (_odp_sched_fn->create_queue(queue->index, &queue->param.sched)) { queue->status = QUEUE_STATUS_FREE; - ODP_ERR("schedule queue init failed\n"); + _ODP_ERR("schedule queue init failed\n"); return ODP_QUEUE_INVALID; } } @@ -370,12 +370,12 @@ static int queue_destroy(odp_queue_t handle) LOCK(queue); if (queue->status == QUEUE_STATUS_FREE) { UNLOCK(queue); - ODP_ERR("queue \"%s\" already free\n", queue->name); + _ODP_ERR("queue \"%s\" already free\n", queue->name); return -1; } if (queue->status == QUEUE_STATUS_DESTROYED) { UNLOCK(queue); - ODP_ERR("queue \"%s\" already destroyed\n", queue->name); + _ODP_ERR("queue \"%s\" already destroyed\n", queue->name); return -1; } @@ -388,7 +388,7 @@ static int queue_destroy(odp_queue_t handle) if (!empty) { UNLOCK(queue); - ODP_ERR("queue \"%s\" not empty\n", queue->name); + _ODP_ERR("queue \"%s\" not empty\n", queue->name); return -1; } if (queue->spsc) @@ -411,7 +411,7 @@ static int queue_destroy(odp_queue_t handle) queue->status = QUEUE_STATUS_DESTROYED; break; default: - ODP_ABORT("Unexpected queue status\n"); + _ODP_ABORT("Unexpected queue status\n"); } if (queue->queue_lf) @@ -528,8 +528,7 @@ static int error_enqueue(odp_queue_t handle, _odp_event_hdr_t *event_hdr) { (void)event_hdr; - ODP_ERR("Enqueue not supported (0x%" PRIx64 ")\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Enqueue not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); return -1; } @@ -541,16 +540,14 @@ static int error_enqueue_multi(odp_queue_t handle, (void)event_hdr; (void)num; - ODP_ERR("Enqueue multi not supported (0x%" PRIx64 ")\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Enqueue multi not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); return -1; } static _odp_event_hdr_t *error_dequeue(odp_queue_t handle) { - ODP_ERR("Dequeue not supported (0x%" PRIx64 ")\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Dequeue not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); return NULL; } @@ -561,8 +558,7 @@ static int error_dequeue_multi(odp_queue_t handle, (void)event_hdr; (void)num; - ODP_ERR("Dequeue multi not supported (0x%" PRIx64 ")\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Dequeue multi not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); return -1; } @@ -587,15 +583,14 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info) int status; if (odp_unlikely(info == NULL)) { - ODP_ERR("Unable to store info, NULL ptr given\n"); + _ODP_ERR("Unable to store info, NULL ptr given\n"); return -1; } queue_id = queue_to_index(handle); if (odp_unlikely(queue_id >= CONFIG_MAX_QUEUES)) { - ODP_ERR("Invalid queue handle: 0x%" PRIx64 "\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Invalid queue handle: 0x%" PRIx64 "\n", odp_queue_to_u64(handle)); return -1; } @@ -607,7 +602,7 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info) if (odp_unlikely(status == QUEUE_STATUS_FREE || status == QUEUE_STATUS_DESTROYED)) { UNLOCK(queue); - ODP_ERR("Invalid queue status:%d\n", status); + _ODP_ERR("Invalid queue status:%d\n", status); return -1; } @@ -630,8 +625,7 @@ static void queue_print(odp_queue_t handle) queue_id = queue_to_index(handle); if (odp_unlikely(queue_id >= CONFIG_MAX_QUEUES)) { - ODP_ERR("Invalid queue handle: 0x%" PRIx64 "\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Invalid queue handle: 0x%" PRIx64 "\n", odp_queue_to_u64(handle)); return; } @@ -643,82 +637,82 @@ static void queue_print(odp_queue_t handle) if (odp_unlikely(status == QUEUE_STATUS_FREE || status == QUEUE_STATUS_DESTROYED)) { UNLOCK(queue); - ODP_ERR("Invalid queue status:%d\n", status); + _ODP_ERR("Invalid queue status:%d\n", status); return; } - ODP_PRINT("\nQueue info\n"); - ODP_PRINT("----------\n"); - ODP_PRINT(" handle %p\n", (void *)queue->handle); - ODP_PRINT(" index %" PRIu32 "\n", queue_id); - ODP_PRINT(" name %s\n", queue->name); - ODP_PRINT(" enq mode %s\n", - queue->param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : - (queue->param.enq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : - (queue->param.enq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : - "unknown"))); - ODP_PRINT(" deq mode %s\n", - queue->param.deq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : - (queue->param.deq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : - (queue->param.deq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : - "unknown"))); - ODP_PRINT(" non-blocking %s\n", - queue->param.nonblocking == ODP_BLOCKING ? "ODP_BLOCKING" : - (queue->param.nonblocking == ODP_NONBLOCKING_LF ? "ODP_NONBLOCKING_LF" : - (queue->param.nonblocking == ODP_NONBLOCKING_WF ? "ODP_NONBLOCKING_WF" : - "unknown"))); - ODP_PRINT(" type %s\n", - queue->type == ODP_QUEUE_TYPE_PLAIN ? "ODP_QUEUE_TYPE_PLAIN" : - (queue->type == ODP_QUEUE_TYPE_SCHED ? "ODP_QUEUE_TYPE_SCHED" : "unknown")); + _ODP_PRINT("\nQueue info\n"); + _ODP_PRINT("----------\n"); + _ODP_PRINT(" handle %p\n", (void *)queue->handle); + _ODP_PRINT(" index %" PRIu32 "\n", queue_id); + _ODP_PRINT(" name %s\n", queue->name); + _ODP_PRINT(" enq mode %s\n", + queue->param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : + (queue->param.enq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : + (queue->param.enq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : + "unknown"))); + _ODP_PRINT(" deq mode %s\n", + queue->param.deq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : + (queue->param.deq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : + (queue->param.deq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : + "unknown"))); + _ODP_PRINT(" non-blocking %s\n", + queue->param.nonblocking == ODP_BLOCKING ? "ODP_BLOCKING" : + (queue->param.nonblocking == ODP_NONBLOCKING_LF ? "ODP_NONBLOCKING_LF" : + (queue->param.nonblocking == ODP_NONBLOCKING_WF ? "ODP_NONBLOCKING_WF" : + "unknown"))); + _ODP_PRINT(" type %s\n", + queue->type == ODP_QUEUE_TYPE_PLAIN ? "ODP_QUEUE_TYPE_PLAIN" : + (queue->type == ODP_QUEUE_TYPE_SCHED ? "ODP_QUEUE_TYPE_SCHED" : "unknown")); if (queue->type == ODP_QUEUE_TYPE_SCHED) { - ODP_PRINT(" sync %s\n", - queue->param.sched.sync == ODP_SCHED_SYNC_PARALLEL ? - "ODP_SCHED_SYNC_PARALLEL" : - (queue->param.sched.sync == ODP_SCHED_SYNC_ATOMIC ? - "ODP_SCHED_SYNC_ATOMIC" : - (queue->param.sched.sync == ODP_SCHED_SYNC_ORDERED ? - "ODP_SCHED_SYNC_ORDERED" : "unknown"))); + _ODP_PRINT(" sync %s\n", + queue->param.sched.sync == ODP_SCHED_SYNC_PARALLEL ? + "ODP_SCHED_SYNC_PARALLEL" : + (queue->param.sched.sync == ODP_SCHED_SYNC_ATOMIC ? + "ODP_SCHED_SYNC_ATOMIC" : + (queue->param.sched.sync == ODP_SCHED_SYNC_ORDERED ? + "ODP_SCHED_SYNC_ORDERED" : "unknown"))); prio = queue->param.sched.prio; - ODP_PRINT(" priority %i (%i in API)\n", max_prio - prio, prio); - ODP_PRINT(" group %i\n", queue->param.sched.group); + _ODP_PRINT(" priority %i (%i in API)\n", max_prio - prio, prio); + _ODP_PRINT(" group %i\n", queue->param.sched.group); if (_odp_sched_id == _ODP_SCHED_ID_BASIC) - ODP_PRINT(" spread %i\n", _odp_sched_basic_get_spread(queue_id)); + _ODP_PRINT(" spread %i\n", _odp_sched_basic_get_spread(queue_id)); } if (queue->pktin.pktio != ODP_PKTIO_INVALID) { if (!odp_pktio_info(queue->pktin.pktio, &pktio_info)) - ODP_PRINT(" pktin %s\n", pktio_info.name); + _ODP_PRINT(" pktin %s\n", pktio_info.name); } if (queue->pktout.pktio != ODP_PKTIO_INVALID) { if (!odp_pktio_info(queue->pktout.pktio, &pktio_info)) - ODP_PRINT(" pktout %s\n", pktio_info.name); + _ODP_PRINT(" pktout %s\n", pktio_info.name); } - ODP_PRINT(" timers %" PRIu64 "\n", - odp_atomic_load_u64(&queue->num_timers)); - ODP_PRINT(" status %s\n", - queue->status == QUEUE_STATUS_READY ? "ready" : - (queue->status == QUEUE_STATUS_NOTSCHED ? "not scheduled" : - (queue->status == QUEUE_STATUS_SCHED ? "scheduled" : "unknown"))); - ODP_PRINT(" param.size %" PRIu32 "\n", queue->param.size); + _ODP_PRINT(" timers %" PRIu64 "\n", + odp_atomic_load_u64(&queue->num_timers)); + _ODP_PRINT(" status %s\n", + queue->status == QUEUE_STATUS_READY ? "ready" : + (queue->status == QUEUE_STATUS_NOTSCHED ? "not scheduled" : + (queue->status == QUEUE_STATUS_SCHED ? "scheduled" : "unknown"))); + _ODP_PRINT(" param.size %" PRIu32 "\n", queue->param.size); if (queue->queue_lf) { - ODP_PRINT(" implementation queue_lf\n"); - ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", - _odp_queue_lf_length(queue->queue_lf), _odp_queue_lf_max_length()); + _ODP_PRINT(" implementation queue_lf\n"); + _ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", + _odp_queue_lf_length(queue->queue_lf), _odp_queue_lf_max_length()); } else if (queue->spsc) { - ODP_PRINT(" implementation ring_spsc\n"); - ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", - ring_spsc_length(queue->ring_spsc), - ring_spsc_max_length(queue->ring_spsc)); + _ODP_PRINT(" implementation ring_spsc\n"); + _ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", + ring_spsc_length(queue->ring_spsc), + ring_spsc_max_length(queue->ring_spsc)); } else if (queue->type == ODP_QUEUE_TYPE_SCHED) { - ODP_PRINT(" implementation ring_st\n"); - ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", - ring_st_length(queue->ring_st), - ring_st_max_length(queue->ring_st)); + _ODP_PRINT(" implementation ring_st\n"); + _ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", + ring_st_length(queue->ring_st), + ring_st_max_length(queue->ring_st)); } else { - ODP_PRINT(" implementation ring_mpmc\n"); - ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", - ring_mpmc_length(queue->ring_mpmc), - ring_mpmc_max_length(queue->ring_mpmc)); + _ODP_PRINT(" implementation ring_mpmc\n"); + _ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", + ring_mpmc_length(queue->ring_mpmc), + ring_mpmc_max_length(queue->ring_mpmc)); } - ODP_PRINT("\n"); + _ODP_PRINT("\n"); UNLOCK(queue); } @@ -742,14 +736,14 @@ static void queue_print_all(void) odp_schedule_sync_t sync = ODP_SCHED_SYNC_PARALLEL; odp_schedule_group_t grp = ODP_SCHED_GROUP_INVALID; - ODP_PRINT("\nList of all queues\n"); - ODP_PRINT("------------------\n"); - ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio grp", - col_width, "name"); + _ODP_PRINT("\nList of all queues\n"); + _ODP_PRINT("------------------\n"); + _ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio grp", + col_width, "name"); if (_odp_sched_id == _ODP_SCHED_ID_BASIC) - ODP_PRINT(" spr\n"); + _ODP_PRINT(" spr\n"); else - ODP_PRINT("\n"); + _ODP_PRINT("\n"); for (i = 0; i < CONFIG_MAX_QUEUES; i++) { queue_entry_t *queue = qentry_from_index(i); @@ -808,9 +802,9 @@ static void queue_print_all(void) order_c = (order == ODP_QUEUE_ORDER_KEEP) ? 'K' : 'I'; - ODP_PRINT("%4u %-*s %c %2s %2s", index, col_width, name, type_c, - status_str, bl_str); - ODP_PRINT(" %c %c %c %6u %6u", enq_c, deq_c, order_c, len, max_len); + _ODP_PRINT("%4u %-*s %c %2s %2s", index, col_width, name, type_c, + status_str, bl_str); + _ODP_PRINT(" %c %c %c %6u %6u", enq_c, deq_c, order_c, len, max_len); if (type == ODP_QUEUE_TYPE_SCHED) { sync_c = (sync == ODP_SCHED_SYNC_PARALLEL) ? 'P' : @@ -818,16 +812,16 @@ static void queue_print_all(void) /* Print prio level matching odp_schedule_print() output */ prio = odp_schedule_max_prio() - prio; - ODP_PRINT(" %c %4i %3i", sync_c, prio, grp); + _ODP_PRINT(" %c %4i %3i", sync_c, prio, grp); if (_odp_sched_id == _ODP_SCHED_ID_BASIC) - ODP_PRINT(" %3i", spr); + _ODP_PRINT(" %3i", spr); } - ODP_PRINT("\n"); + _ODP_PRINT("\n"); } - ODP_PRINT("\n"); + _ODP_PRINT("\n"); } static inline int _sched_queue_enq_multi(odp_queue_t handle, @@ -863,7 +857,7 @@ static inline int _sched_queue_enq_multi(odp_queue_t handle, /* Add queue to scheduling */ if (sched && _odp_sched_fn->sched_queue(queue->index)) - ODP_ABORT("schedule_queue failed\n"); + _ODP_ABORT("schedule_queue failed\n"); return num_enq; } @@ -990,7 +984,7 @@ static int queue_init(queue_entry_t *queue, const char *name, queue_size = MIN_QUEUE_SIZE; if (queue_size > _odp_queue_glb->config.max_queue_size) { - ODP_ERR("Too large queue size %u\n", queue_size); + _ODP_ERR("Too large queue size %u\n", queue_size); return -1; } @@ -1029,7 +1023,7 @@ static int queue_init(queue_entry_t *queue, const char *name, queue->ring_mpmc = ring_mpmc_create(queue->name, queue_size); if (queue->ring_mpmc == NULL) { - ODP_ERR("Creating MPMC ring failed\n"); + _ODP_ERR("Creating MPMC ring failed\n"); return -1; } } else { @@ -1038,7 +1032,7 @@ static int queue_init(queue_entry_t *queue, const char *name, queue->ring_st = ring_st_create(queue->name, queue_size); if (queue->ring_st == NULL) { - ODP_ERR("Creating ST ring failed\n"); + _ODP_ERR("Creating ST ring failed\n"); return -1; } } diff --git a/platform/linux-dpdk/odp_queue_eventdev.c b/platform/linux-dpdk/odp_queue_eventdev.c index 600730f56..c0928bfd2 100644 --- a/platform/linux-dpdk/odp_queue_eventdev.c +++ b/platform/linux-dpdk/odp_queue_eventdev.c @@ -79,7 +79,7 @@ static uint8_t event_queue_ids(odp_schedule_sync_t sync, uint8_t *first_id) if (sync == ODP_SCHED_SYNC_ORDERED) return _odp_eventdev_gbl->event_queue.num_ordered; - ODP_ABORT("Invalid schedule sync type\n"); + _ODP_ABORT("Invalid schedule sync type\n"); return 0; } @@ -88,38 +88,38 @@ static int read_config_file(eventdev_global_t *eventdev) const char *str; int val = 0; - ODP_PRINT("\nScheduler config\n----------------\n"); + _ODP_PRINT("\nScheduler config\n----------------\n"); str = "sched_eventdev.num_atomic_queues"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } - ODP_PRINT("%s: %i\n", str, val); + _ODP_PRINT("%s: %i\n", str, val); eventdev->event_queue.num_atomic = val; str = "sched_eventdev.num_ordered_queues"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } - ODP_PRINT("%s: %i\n", str, val); + _ODP_PRINT("%s: %i\n", str, val); eventdev->event_queue.num_ordered = val; str = "sched_eventdev.num_parallel_queues"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } - ODP_PRINT("%s: %i\n\n", str, val); + _ODP_PRINT("%s: %i\n\n", str, val); eventdev->event_queue.num_parallel = val; str = "sched_eventdev.num_ports"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } - ODP_PRINT("%s: %i\n\n", str, val); + _ODP_PRINT("%s: %i\n\n", str, val); eventdev->num_event_ports = val; return 0; @@ -141,34 +141,34 @@ static int queue_capa(odp_queue_capability_t *capa, int sched ODP_UNUSED) static void print_dev_info(const struct rte_event_dev_info *info) { - ODP_PRINT("\nEvent device info\n" - "-----------------\n" - "driver name: %s\n" - "min_dequeue_timeout_ns: %" PRIu32 "\n" - "max_dequeue_timeout_ns: %" PRIu32 "\n" - "dequeue_timeout_ns: %" PRIu32 "\n" - "max_event_queues: %" PRIu8 "\n" - "max_event_queue_flows: %" PRIu32 "\n" - "max_event_queue_priority_levels: %" PRIu8 "\n" - "max_event_priority_levels: %" PRIu8 "\n" - "max_event_ports: %" PRIu8 "\n" - "max_event_port_dequeue_depth: %" PRIu8 "\n" - "max_event_port_enqueue_depth: %" PRIu32 "\n" - "max_num_events: %" PRId32 "\n" - "event_dev_cap: %" PRIu32 "\n", - info->driver_name, - info->min_dequeue_timeout_ns, - info->max_dequeue_timeout_ns, - info->dequeue_timeout_ns, - info->max_event_queues, - info->max_event_queue_flows, - info->max_event_queue_priority_levels, - info->max_event_priority_levels, - info->max_event_ports, - info->max_event_port_dequeue_depth, - info->max_event_port_enqueue_depth, - info->max_num_events, - info->event_dev_cap); + _ODP_PRINT("\nEvent device info\n" + "-----------------\n" + "driver name: %s\n" + "min_dequeue_timeout_ns: %" PRIu32 "\n" + "max_dequeue_timeout_ns: %" PRIu32 "\n" + "dequeue_timeout_ns: %" PRIu32 "\n" + "max_event_queues: %" PRIu8 "\n" + "max_event_queue_flows: %" PRIu32 "\n" + "max_event_queue_priority_levels: %" PRIu8 "\n" + "max_event_priority_levels: %" PRIu8 "\n" + "max_event_ports: %" PRIu8 "\n" + "max_event_port_dequeue_depth: %" PRIu8 "\n" + "max_event_port_enqueue_depth: %" PRIu32 "\n" + "max_num_events: %" PRId32 "\n" + "event_dev_cap: %" PRIu32 "\n", + info->driver_name, + info->min_dequeue_timeout_ns, + info->max_dequeue_timeout_ns, + info->dequeue_timeout_ns, + info->max_event_queues, + info->max_event_queue_flows, + info->max_event_queue_priority_levels, + info->max_event_priority_levels, + info->max_event_ports, + info->max_event_port_dequeue_depth, + info->max_event_port_enqueue_depth, + info->max_num_events, + info->event_dev_cap); } int _odp_service_setup(uint32_t service_id) @@ -180,7 +180,7 @@ int _odp_service_setup(uint32_t service_id) int32_t min_num_serv = INT32_MAX; if (!rte_service_lcore_count()) { - ODP_ERR("No service cores available\n"); + _ODP_ERR("No service cores available\n"); return -1; } @@ -195,7 +195,7 @@ int _odp_service_setup(uint32_t service_id) } } if (rte_service_map_lcore_set(service_id, lcore, 1)) { - ODP_ERR("Unable to map service to core\n"); + _ODP_ERR("Unable to map service to core\n"); return -1; } return 0; @@ -239,7 +239,7 @@ static int setup_queues(uint8_t dev_id, uint8_t first_queue_id, queue->sync = sync; if (rte_event_queue_default_conf_get(dev_id, i, &queue_conf)) { - ODP_ERR("rte_event_queue_default_conf_get failed\n"); + _ODP_ERR("rte_event_queue_default_conf_get failed\n"); return -1; } queue_conf.schedule_type = event_schedule_type(sync); @@ -250,7 +250,7 @@ static int setup_queues(uint8_t dev_id, uint8_t first_queue_id, queue_conf.nb_atomic_flows = num_flows; if (rte_event_queue_setup(dev_id, i, &queue_conf)) { - ODP_ERR("rte_event_queue_setup failed\n"); + _ODP_ERR("rte_event_queue_setup failed\n"); return -1; } } @@ -314,7 +314,7 @@ int _odp_dummy_link_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int nu ret = rte_event_port_link(dev_id, 0, &queue_id, &priority, 1); if (ret != 1) { - ODP_ERR("rte_event_port_link failed: %d\n", ret); + _ODP_ERR("rte_event_port_link failed: %d\n", ret); return -1; } dummy_linked_queues[num_linked++] = queue_id; @@ -330,7 +330,7 @@ int _odp_dummy_unlink_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int for (i = 0; i < num; i++) { if (rte_event_port_unlink(dev_id, 0, &dummy_linked_queues[i], 1) < 0) { - ODP_ERR("rte_event_port_unlink failed\n"); + _ODP_ERR("rte_event_port_unlink failed\n"); return -1; } } @@ -345,7 +345,7 @@ static int configure_ports(uint8_t dev_id, for (i = 0; i < dev_conf->nb_event_ports; i++) { if (rte_event_port_default_conf_get(dev_id, i, &port_conf)) { - ODP_ERR("rte_event_port_default_conf_get failed\n"); + _ODP_ERR("rte_event_port_default_conf_get failed\n"); return -1; } @@ -354,7 +354,7 @@ static int configure_ports(uint8_t dev_id, port_conf.enqueue_depth = dev_conf->nb_event_port_enqueue_depth; if (rte_event_port_setup(dev_id, i, &port_conf)) { - ODP_ERR("rte_event_port_setup failed\n"); + _ODP_ERR("rte_event_port_setup failed\n"); return -1; } } @@ -374,7 +374,7 @@ static int init_event_dev(void) int i; if (rte_event_dev_count() < 1) { - ODP_ERR("No eventdev devices found\n"); + _ODP_ERR("No eventdev devices found\n"); return -1; } @@ -392,7 +392,7 @@ static int init_event_dev(void) _odp_eventdev_gbl->port[i].linked = 0; if (rte_event_dev_info_get(dev_id, &info)) { - ODP_ERR("rte_event_dev_info_get failed\n"); + _ODP_ERR("rte_event_dev_info_get failed\n"); return -1; } print_dev_info(&info); @@ -400,7 +400,7 @@ static int init_event_dev(void) _odp_eventdev_gbl->num_prio = RTE_MIN(NUM_PRIO, info.max_event_queue_priority_levels); if (!(info.event_dev_cap & RTE_EVENT_DEV_CAP_QUEUE_QOS)) { - ODP_PRINT(" Only one QoS level supported!\n"); + _ODP_PRINT(" Only one QoS level supported!\n"); _odp_eventdev_gbl->num_prio = 1; } @@ -433,19 +433,19 @@ static int init_event_dev(void) ret = rte_event_dev_configure(dev_id, &config); if (ret < 0) { - ODP_ERR("rte_event_dev_configure failed\n"); + _ODP_ERR("rte_event_dev_configure failed\n"); return -1; } _odp_eventdev_gbl->config = config; _odp_eventdev_gbl->num_event_ports = config.nb_event_ports; if (configure_ports(dev_id, &config)) { - ODP_ERR("Configuring eventdev ports failed\n"); + _ODP_ERR("Configuring eventdev ports failed\n"); return -1; } if (configure_queues(dev_id, num_flows)) { - ODP_ERR("Configuring eventdev queues failed\n"); + _ODP_ERR("Configuring eventdev queues failed\n"); return -1; } @@ -459,17 +459,17 @@ static int init_event_dev(void) ret = rte_event_dev_service_id_get(dev_id, &service_id); if (ret) { - ODP_ERR("Unable to retrieve service ID\n"); + _ODP_ERR("Unable to retrieve service ID\n"); return -1; } if (_odp_service_setup(service_id)) { - ODP_ERR("Failed to setup service core\n"); + _ODP_ERR("Failed to setup service core\n"); return -1; } } if (rte_event_dev_start(dev_id)) { - ODP_ERR("rte_event_dev_start failed\n"); + _ODP_ERR("rte_event_dev_start failed\n"); return -1; } @@ -506,7 +506,7 @@ static int queue_init_global(void) odp_shm_t shm; odp_queue_capability_t capa; - ODP_DBG("Queue init global\n"); + _ODP_DBG("Queue init global\n"); /* Fill in queue entry field offsets for inline functions */ memset(&_odp_queue_inline_offset, 0, @@ -543,11 +543,11 @@ static int queue_init_global(void) queue_capa(&capa, 0); - ODP_DBG(" queue_entry_t size %zu\n", sizeof(queue_entry_t)); - ODP_DBG(" max num queues %u\n", capa.max_queues); - ODP_DBG(" max plain queue size %u\n", capa.plain.max_size); - ODP_DBG(" max num lockfree %u\n", capa.plain.lockfree.max_num); - ODP_DBG(" max lockfree size %u\n\n", capa.plain.lockfree.max_size); + _ODP_DBG(" queue_entry_t size %zu\n", sizeof(queue_entry_t)); + _ODP_DBG(" max num queues %u\n", capa.max_queues); + _ODP_DBG(" max plain queue size %u\n", capa.plain.max_size); + _ODP_DBG(" max num lockfree %u\n", capa.plain.lockfree.max_num); + _ODP_DBG(" max lockfree size %u\n\n", capa.plain.lockfree.max_size); return 0; } @@ -558,7 +558,7 @@ static int queue_init_local(void) memset(&_odp_eventdev_local, 0, sizeof(eventdev_local_t)); - ODP_ASSERT(thread_id <= UINT8_MAX); + _ODP_ASSERT(thread_id <= UINT8_MAX); _odp_eventdev_local.port_id = thread_id; _odp_eventdev_local.paused = 0; _odp_eventdev_local.started = 0; @@ -581,7 +581,7 @@ static int queue_term_global(void) queue = qentry_from_index(i); LOCK(queue); if (queue->status != QUEUE_STATUS_FREE) { - ODP_ERR("Not destroyed queue: %s\n", queue->name); + _ODP_ERR("Not destroyed queue: %s\n", queue->name); ret = -1; } UNLOCK(queue); @@ -596,12 +596,12 @@ static int queue_term_global(void) sleep(1); if (rte_event_dev_close(_odp_eventdev_gbl->dev_id)) { - ODP_ERR("Failed to close event device\n"); + _ODP_ERR("Failed to close event device\n"); ret = -1; } if (odp_shm_free(_odp_eventdev_gbl->shm)) { - ODP_ERR("Shm free failed for evendev\n"); + _ODP_ERR("Shm free failed for evendev\n"); ret = -1; } @@ -661,7 +661,7 @@ static odp_queue_t queue_create(const char *name, if (type == ODP_QUEUE_TYPE_SCHED) { if (param->sched.prio < odp_schedule_min_prio() || param->sched.prio > odp_schedule_max_prio()) { - ODP_ERR("Bad queue priority: %i\n", param->sched.prio); + _ODP_ERR("Bad queue priority: %i\n", param->sched.prio); return ODP_QUEUE_INVALID; } if (param->size > _odp_eventdev_gbl->sched_config.max_queue_size) @@ -702,7 +702,7 @@ static odp_queue_t queue_create(const char *name, if (queue->status == QUEUE_STATUS_FREE) { if (queue_init(queue, name, param)) { UNLOCK(queue); - ODP_ERR("Queue init failed\n"); + _ODP_ERR("Queue init failed\n"); return ODP_QUEUE_INVALID; } @@ -718,7 +718,7 @@ static odp_queue_t queue_create(const char *name, } if (handle == ODP_QUEUE_INVALID) { - ODP_ERR("No free queues left\n"); + _ODP_ERR("No free queues left\n"); return ODP_QUEUE_INVALID; } @@ -726,7 +726,7 @@ static odp_queue_t queue_create(const char *name, if (_odp_sched_fn->create_queue(queue->index, &queue->param.sched)) { queue->status = QUEUE_STATUS_FREE; - ODP_ERR("schedule queue init failed\n"); + _ODP_ERR("schedule queue init failed\n"); return ODP_QUEUE_INVALID; } } @@ -746,13 +746,13 @@ static int queue_destroy(odp_queue_t handle) LOCK(queue); if (queue->status == QUEUE_STATUS_FREE) { UNLOCK(queue); - ODP_ERR("queue \"%s\" already free\n", queue->name); + _ODP_ERR("queue \"%s\" already free\n", queue->name); return -1; } if (queue->type == ODP_QUEUE_TYPE_PLAIN) { if (ring_mpmc_is_empty(queue->ring_mpmc) == 0) { UNLOCK(queue); - ODP_ERR("queue \"%s\" not empty\n", queue->name); + _ODP_ERR("queue \"%s\" not empty\n", queue->name); return -1; } ring_mpmc_free(queue->ring_mpmc); @@ -767,7 +767,7 @@ static int queue_destroy(odp_queue_t handle) _odp_sched_fn->destroy_queue(queue->index); break; default: - ODP_ABORT("Unexpected queue status\n"); + _ODP_ABORT("Unexpected queue status\n"); } UNLOCK(queue); @@ -877,8 +877,7 @@ static int error_enqueue(odp_queue_t handle, _odp_event_hdr_t *event_hdr) { (void)event_hdr; - ODP_ERR("Enqueue not supported (0x%" PRIx64 ")\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Enqueue not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); return -1; } @@ -890,16 +889,14 @@ static int error_enqueue_multi(odp_queue_t handle, (void)event_hdr; (void)num; - ODP_ERR("Enqueue multi not supported (0x%" PRIx64 ")\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Enqueue multi not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); return -1; } static _odp_event_hdr_t *error_dequeue(odp_queue_t handle) { - ODP_ERR("Dequeue not supported (0x%" PRIx64 ")\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Dequeue not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); return NULL; } @@ -910,8 +907,7 @@ static int error_dequeue_multi(odp_queue_t handle, (void)event_hdr; (void)num; - ODP_ERR("Dequeue multi not supported (0x%" PRIx64 ")\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Dequeue multi not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); return -1; } @@ -935,15 +931,14 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info) int status; if (odp_unlikely(info == NULL)) { - ODP_ERR("Unable to store info, NULL ptr given\n"); + _ODP_ERR("Unable to store info, NULL ptr given\n"); return -1; } queue_id = queue_to_index(handle); if (odp_unlikely(queue_id >= CONFIG_MAX_QUEUES)) { - ODP_ERR("Invalid queue handle: 0x%" PRIx64 "\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Invalid queue handle: 0x%" PRIx64 "\n", odp_queue_to_u64(handle)); return -1; } @@ -954,7 +949,7 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info) if (odp_unlikely(status == QUEUE_STATUS_FREE)) { UNLOCK(queue); - ODP_ERR("Invalid queue status:%d\n", status); + _ODP_ERR("Invalid queue status:%d\n", status); return -1; } @@ -976,8 +971,7 @@ static void queue_print(odp_queue_t handle) queue_id = queue_to_index(handle); if (odp_unlikely(queue_id >= CONFIG_MAX_QUEUES)) { - ODP_ERR("Invalid queue handle: 0x%" PRIx64 "\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Invalid queue handle: 0x%" PRIx64 "\n", odp_queue_to_u64(handle)); return; } @@ -988,66 +982,66 @@ static void queue_print(odp_queue_t handle) if (odp_unlikely(status == QUEUE_STATUS_FREE)) { UNLOCK(queue); - ODP_ERR("Invalid queue status:%d\n", status); + _ODP_ERR("Invalid queue status:%d\n", status); return; } - ODP_PRINT("\nQueue info\n"); - ODP_PRINT("----------\n"); - ODP_PRINT(" handle %p\n", (void *)handle); - ODP_PRINT(" index %" PRIu32 "\n", queue->index); - ODP_PRINT(" name %s\n", queue->name); - ODP_PRINT(" enq mode %s\n", - queue->param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : - (queue->param.enq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : - (queue->param.enq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : - "unknown"))); - ODP_PRINT(" deq mode %s\n", - queue->param.deq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : - (queue->param.deq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : - (queue->param.deq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : - "unknown"))); - ODP_PRINT(" non-blocking %s\n", - queue->param.nonblocking == ODP_BLOCKING ? "ODP_BLOCKING" : - (queue->param.nonblocking == ODP_NONBLOCKING_LF ? "ODP_NONBLOCKING_LF" : - (queue->param.nonblocking == ODP_NONBLOCKING_WF ? "ODP_NONBLOCKING_WF" : - "unknown"))); - ODP_PRINT(" type %s\n", - queue->type == ODP_QUEUE_TYPE_PLAIN ? "ODP_QUEUE_TYPE_PLAIN" : - (queue->type == ODP_QUEUE_TYPE_SCHED ? "ODP_QUEUE_TYPE_SCHED" : "unknown")); + _ODP_PRINT("\nQueue info\n"); + _ODP_PRINT("----------\n"); + _ODP_PRINT(" handle %p\n", (void *)handle); + _ODP_PRINT(" index %" PRIu32 "\n", queue->index); + _ODP_PRINT(" name %s\n", queue->name); + _ODP_PRINT(" enq mode %s\n", + queue->param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : + (queue->param.enq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : + (queue->param.enq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : + "unknown"))); + _ODP_PRINT(" deq mode %s\n", + queue->param.deq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : + (queue->param.deq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : + (queue->param.deq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : + "unknown"))); + _ODP_PRINT(" non-blocking %s\n", + queue->param.nonblocking == ODP_BLOCKING ? "ODP_BLOCKING" : + (queue->param.nonblocking == ODP_NONBLOCKING_LF ? "ODP_NONBLOCKING_LF" : + (queue->param.nonblocking == ODP_NONBLOCKING_WF ? "ODP_NONBLOCKING_WF" : + "unknown"))); + _ODP_PRINT(" type %s\n", + queue->type == ODP_QUEUE_TYPE_PLAIN ? "ODP_QUEUE_TYPE_PLAIN" : + (queue->type == ODP_QUEUE_TYPE_SCHED ? "ODP_QUEUE_TYPE_SCHED" : "unknown")); if (queue->type == ODP_QUEUE_TYPE_SCHED) { - ODP_PRINT(" sync %s\n", - queue->param.sched.sync == ODP_SCHED_SYNC_PARALLEL ? - "ODP_SCHED_SYNC_PARALLEL" : - (queue->param.sched.sync == ODP_SCHED_SYNC_ATOMIC ? - "ODP_SCHED_SYNC_ATOMIC" : - (queue->param.sched.sync == ODP_SCHED_SYNC_ORDERED ? - "ODP_SCHED_SYNC_ORDERED" : "unknown"))); - ODP_PRINT(" priority %d\n", queue->param.sched.prio); - ODP_PRINT(" group %d\n", queue->param.sched.group); + _ODP_PRINT(" sync %s\n", + queue->param.sched.sync == ODP_SCHED_SYNC_PARALLEL ? + "ODP_SCHED_SYNC_PARALLEL" : + (queue->param.sched.sync == ODP_SCHED_SYNC_ATOMIC ? + "ODP_SCHED_SYNC_ATOMIC" : + (queue->param.sched.sync == ODP_SCHED_SYNC_ORDERED ? + "ODP_SCHED_SYNC_ORDERED" : "unknown"))); + _ODP_PRINT(" priority %d\n", queue->param.sched.prio); + _ODP_PRINT(" group %d\n", queue->param.sched.group); } if (queue->pktin.pktio != ODP_PKTIO_INVALID) { if (!odp_pktio_info(queue->pktin.pktio, &pktio_info)) - ODP_PRINT(" pktin %s\n", pktio_info.name); + _ODP_PRINT(" pktin %s\n", pktio_info.name); } if (queue->pktout.pktio != ODP_PKTIO_INVALID) { if (!odp_pktio_info(queue->pktout.pktio, &pktio_info)) - ODP_PRINT(" pktout %s\n", pktio_info.name); + _ODP_PRINT(" pktout %s\n", pktio_info.name); } - ODP_PRINT(" timers %" PRIu64 "\n", - odp_atomic_load_u64(&queue->num_timers)); - ODP_PRINT(" status %s\n", - queue->status == QUEUE_STATUS_READY ? "ready" : - (queue->status == QUEUE_STATUS_SCHED ? "scheduled" : "clearunknown")); - ODP_PRINT(" param.size %" PRIu32 "\n", queue->param.size); + _ODP_PRINT(" timers %" PRIu64 "\n", + odp_atomic_load_u64(&queue->num_timers)); + _ODP_PRINT(" status %s\n", + queue->status == QUEUE_STATUS_READY ? "ready" : + (queue->status == QUEUE_STATUS_SCHED ? "scheduled" : "clearunknown")); + _ODP_PRINT(" param.size %" PRIu32 "\n", queue->param.size); if (queue->type == ODP_QUEUE_TYPE_PLAIN) { - ODP_PRINT(" implementation ring_mpmc\n"); - ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", - ring_mpmc_length(queue->ring_mpmc), - ring_mpmc_max_length(queue->ring_mpmc)); + _ODP_PRINT(" implementation ring_mpmc\n"); + _ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", + ring_mpmc_length(queue->ring_mpmc), + ring_mpmc_max_length(queue->ring_mpmc)); } else { - ODP_PRINT(" implementation eventdev\n"); + _ODP_PRINT(" implementation eventdev\n"); } - ODP_PRINT("\n"); + _ODP_PRINT("\n"); UNLOCK(queue); } @@ -1071,7 +1065,7 @@ static inline int _sched_queue_enq_multi(odp_queue_t handle, if (odp_unlikely(queue->status != QUEUE_STATUS_SCHED)) { UNLOCK(queue); - ODP_ERR("Bad queue status\n"); + _ODP_ERR("Bad queue status\n"); return -1; } @@ -1082,8 +1076,8 @@ static inline int _sched_queue_enq_multi(odp_queue_t handle, UNLOCK(queue); if (odp_unlikely(port_id >= _odp_eventdev_gbl->num_event_ports)) { - ODP_ERR("Max %" PRIu8 " scheduled workers supported\n", - _odp_eventdev_gbl->num_event_ports); + _ODP_ERR("Max %" PRIu8 " scheduled workers supported\n", + _odp_eventdev_gbl->num_event_ports); return 0; } @@ -1161,7 +1155,7 @@ static int queue_init(queue_entry_t *queue, const char *name, if (queue_type == ODP_QUEUE_TYPE_PLAIN && queue_size > _odp_eventdev_gbl->plain_config.max_queue_size) { - ODP_ERR("Too large queue size %u\n", queue_size); + _ODP_ERR("Too large queue size %u\n", queue_size); return -1; } @@ -1188,7 +1182,7 @@ static int queue_init(queue_entry_t *queue, const char *name, queue->ring_mpmc = ring_mpmc_create(queue->name, queue_size); if (queue->ring_mpmc == NULL) { - ODP_ERR("Creating MPMC ring failed\n"); + _ODP_ERR("Creating MPMC ring failed\n"); return -1; } } else { diff --git a/platform/linux-dpdk/odp_queue_if.c b/platform/linux-dpdk/odp_queue_if.c index 3768dca54..04ac1fd60 100644 --- a/platform/linux-dpdk/odp_queue_if.c +++ b/platform/linux-dpdk/odp_queue_if.c @@ -120,7 +120,7 @@ int _odp_queue_init_global(void) _odp_queue_fn = &_odp_queue_eventdev_fn; _odp_queue_api = &_odp_queue_eventdev_api; } else { - ODP_ABORT("Unknown scheduler specified via ODP_SCHEDULER\n"); + _ODP_ABORT("Unknown scheduler specified via ODP_SCHEDULER\n"); return -1; } diff --git a/platform/linux-dpdk/odp_queue_spsc.c b/platform/linux-dpdk/odp_queue_spsc.c index a5a31498d..61dde0ed0 100644 --- a/platform/linux-dpdk/odp_queue_spsc.c +++ b/platform/linux-dpdk/odp_queue_spsc.c @@ -20,7 +20,7 @@ static inline int spsc_enq_multi(odp_queue_t handle, ring_spsc = queue->ring_spsc; if (odp_unlikely(queue->status < QUEUE_STATUS_READY)) { - ODP_ERR("Bad queue status\n"); + _ODP_ERR("Bad queue status\n"); return -1; } @@ -91,5 +91,5 @@ void _odp_queue_spsc_init(queue_entry_t *queue, uint32_t queue_size) queue->ring_spsc = ring_spsc_create(queue->name, queue_size); if (queue->ring_spsc == NULL) - ODP_ABORT("Creating SPSC ring failed\n"); + _ODP_ABORT("Creating SPSC ring failed\n"); } diff --git a/platform/linux-dpdk/odp_schedule_eventdev.c b/platform/linux-dpdk/odp_schedule_eventdev.c index 605cf0e13..8a6a1dd45 100644 --- a/platform/linux-dpdk/odp_schedule_eventdev.c +++ b/platform/linux-dpdk/odp_schedule_eventdev.c @@ -13,6 +13,8 @@ #include <odp/api/ticketlock.h> #include <odp/api/thrmask.h> +#include <odp/api/plat/schedule_inline_types.h> + #include <odp_config_internal.h> #include <odp_debug_internal.h> #include <odp_eventdev_internal.h> @@ -65,7 +67,7 @@ static int link_port(uint8_t dev_id, uint8_t port_id, uint8_t queue_ids[], ret = rte_event_port_link(dev_id, port_id, queue_ids, priorities, nb_links); if (ret < 0 || (queue_ids && ret != nb_links)) { - ODP_ERR("rte_event_port_link failed: %d\n", ret); + _ODP_ERR("rte_event_port_link failed: %d\n", ret); odp_ticketlock_unlock(&_odp_eventdev_gbl->port_lock); return ret; } @@ -91,7 +93,7 @@ static int unlink_port(uint8_t dev_id, uint8_t port_id, uint8_t queue_ids[], ret = rte_event_port_unlink(dev_id, port_id, queue_ids, nb_links); if (ret < 0) { - ODP_ERR("rte_event_port_unlink failed: %d\n", ret); + _ODP_ERR("rte_event_port_unlink failed: %d\n", ret); odp_ticketlock_unlock(&_odp_eventdev_gbl->port_lock); return ret; } @@ -99,7 +101,7 @@ static int unlink_port(uint8_t dev_id, uint8_t port_id, uint8_t queue_ids[], do { ret = rte_event_port_unlinks_in_progress(dev_id, port_id); if (ret < 0) { - ODP_ERR("rte_event_port_unlinks_in_progress failed: " + _ODP_ERR("rte_event_port_unlinks_in_progress failed: " "%d\n", ret); break; } @@ -197,7 +199,7 @@ static int link_group(int group, const odp_thrmask_t *mask, odp_bool_t unlink) ret = link_port(dev_id, port_id, queue_ids, priorities, nb_links, 0); if (ret < 0) { - ODP_ERR("Modifying port links failed\n"); + _ODP_ERR("Modifying port links failed\n"); return -1; } } @@ -214,7 +216,7 @@ static int rx_adapter_create(uint8_t dev_id, uint8_t rx_adapter_id, ret = rte_event_eth_rx_adapter_caps_get(dev_id, rx_adapter_id, &capa); if (ret) { - ODP_ERR("rte_event_eth_rx_adapter_caps_get failed: %d\n", ret); + _ODP_ERR("rte_event_eth_rx_adapter_caps_get failed: %d\n", ret); return -1; } if ((capa & RTE_EVENT_ETH_RX_ADAPTER_CAP_MULTI_EVENTQ) == 0) @@ -227,7 +229,7 @@ static int rx_adapter_create(uint8_t dev_id, uint8_t rx_adapter_id, ret = rte_event_eth_rx_adapter_create(rx_adapter_id, dev_id, &port_config); if (ret) { - ODP_ERR("rte_event_eth_rx_adapter_create failed: %d\n", ret); + _ODP_ERR("rte_event_eth_rx_adapter_create failed: %d\n", ret); return -1; } @@ -273,7 +275,7 @@ static int rx_adapter_add_queues(uint8_t rx_adapter_id, uint8_t port_id, ret = rte_event_eth_rx_adapter_queue_add(rx_adapter_id, port_id, rx_queue_id, &qconf); if (ret) { - ODP_ERR("rte_event_eth_rx_adapter_queue_add failed\n"); + _ODP_ERR("rte_event_eth_rx_adapter_queue_add failed\n"); return -1; } @@ -299,7 +301,7 @@ int _odp_rx_adapter_close(void) if (_odp_eventdev_gbl->rx_adapter.status != RX_ADAPTER_STOPPED && rte_event_eth_rx_adapter_stop(rx_adapter_id)) { - ODP_ERR("Failed to stop RX adapter\n"); + _ODP_ERR("Failed to stop RX adapter\n"); ret = -1; } @@ -317,14 +319,14 @@ void _odp_rx_adapter_port_stop(uint16_t port_id) uint8_t rx_adapter_id = _odp_eventdev_gbl->rx_adapter.id; if (rte_event_eth_rx_adapter_queue_del(rx_adapter_id, port_id, -1)) - ODP_ERR("Failed to delete RX queue\n"); + _ODP_ERR("Failed to delete RX queue\n"); rte_eth_dev_stop(port_id); } static int schedule_init_global(void) { - ODP_DBG("Using eventdev scheduler\n"); + _ODP_DBG("Using eventdev scheduler\n"); return 0; } @@ -374,7 +376,7 @@ static int schedule_create_queue(uint32_t qi, int thr; if (sched_param->group < 0 || sched_param->group >= NUM_SCHED_GRPS) { - ODP_ERR("Bad schedule group\n"); + _ODP_ERR("Bad schedule group\n"); return -1; } @@ -427,9 +429,8 @@ static void schedule_pktio_start(int pktio_index, int num_pktin, * event queues are linked when rte_event_eth_rx_adapter_queue_add() is * called. */ if (odp_atomic_load_u32(&_odp_eventdev_gbl->num_started)) - ODP_PRINT("All ODP pktio devices used by the scheduler should " - "be started before calling odp_schedule() for the " - "first time.\n"); + _ODP_PRINT("All ODP pktio devices used by the scheduler should " + "be started before calling odp_schedule() for the first time.\n"); _odp_eventdev_gbl->pktio[port_id] = entry; @@ -438,11 +439,11 @@ static void schedule_pktio_start(int pktio_index, int num_pktin, if (_odp_eventdev_gbl->rx_adapter.status == RX_ADAPTER_INIT && rx_adapter_create(_odp_eventdev_gbl->dev_id, rx_adapter_id, &_odp_eventdev_gbl->config)) - ODP_ABORT("Creating eventdev RX adapter failed\n"); + _ODP_ABORT("Creating eventdev RX adapter failed\n"); if (rx_adapter_add_queues(rx_adapter_id, port_id, num_pktin, pktin_idx, queue)) - ODP_ABORT("Adding RX adapter queues failed\n"); + _ODP_ABORT("Adding RX adapter queues failed\n"); if (_odp_eventdev_gbl->rx_adapter.status == RX_ADAPTER_STOPPED) { uint32_t service_id = 0; @@ -451,14 +452,14 @@ static void schedule_pktio_start(int pktio_index, int num_pktin, ret = rte_event_eth_rx_adapter_service_id_get(rx_adapter_id, &service_id); if (ret && ret != -ESRCH) { - ODP_ABORT("Unable to retrieve service ID\n"); + _ODP_ABORT("Unable to retrieve service ID\n"); } else if (!ret) { if (_odp_service_setup(service_id)) - ODP_ABORT("Unable to start RX service\n"); + _ODP_ABORT("Unable to start RX service\n"); } if (rte_event_eth_rx_adapter_start(rx_adapter_id)) - ODP_ABORT("Unable to start RX adapter\n"); + _ODP_ABORT("Unable to start RX adapter\n"); _odp_eventdev_gbl->rx_adapter.status = RX_ADAPTER_RUNNING; } @@ -628,8 +629,8 @@ static inline int schedule_loop(odp_queue_t *out_queue, uint64_t wait, uint8_t port_id = _odp_eventdev_local.port_id; if (odp_unlikely(port_id >= _odp_eventdev_gbl->num_event_ports)) { - ODP_ERR("Max %" PRIu8 " scheduled workers supported\n", - _odp_eventdev_gbl->num_event_ports); + _ODP_ERR("Max %" PRIu8 " scheduled workers supported\n", + _odp_eventdev_gbl->num_event_ports); return 0; } @@ -715,7 +716,7 @@ static void schedule_pause(void) { if (unlink_port(_odp_eventdev_gbl->dev_id, _odp_eventdev_local.port_id, NULL, 0) < 0) - ODP_ERR("Unable to pause scheduling\n"); + _ODP_ERR("Unable to pause scheduling\n"); _odp_eventdev_local.paused = 1; } @@ -723,7 +724,7 @@ static void schedule_pause(void) static void schedule_resume(void) { if (resume_scheduling(_odp_eventdev_gbl->dev_id, _odp_eventdev_local.port_id)) - ODP_ERR("Unable to resume scheduling\n"); + _ODP_ERR("Unable to resume scheduling\n"); _odp_eventdev_local.paused = 0; } @@ -1059,12 +1060,12 @@ static void schedule_print(void) (void)schedule_capability(&capa); - ODP_PRINT("\nScheduler debug info\n"); - ODP_PRINT("--------------------\n"); - ODP_PRINT(" scheduler: eventdev\n"); - ODP_PRINT(" max groups: %u\n", capa.max_groups); - ODP_PRINT(" max priorities: %u\n", capa.max_prios); - ODP_PRINT("\n"); + _ODP_PRINT("\nScheduler debug info\n"); + _ODP_PRINT("--------------------\n"); + _ODP_PRINT(" scheduler: eventdev\n"); + _ODP_PRINT(" max groups: %u\n", capa.max_groups); + _ODP_PRINT(" max priorities: %u\n", capa.max_prios); + _ODP_PRINT("\n"); } /* Fill in scheduler interface */ @@ -1088,7 +1089,7 @@ const schedule_fn_t _odp_schedule_eventdev_fn = { }; /* Fill in scheduler API calls */ -const schedule_api_t _odp_schedule_eventdev_api = { +const _odp_schedule_api_fn_t _odp_schedule_eventdev_api = { .schedule_wait_time = schedule_wait_time, .schedule_capability = schedule_capability, .schedule_config_init = schedule_config_init, diff --git a/platform/linux-dpdk/odp_schedule_if.c b/platform/linux-dpdk/odp_schedule_if.c index d5b39b656..fa6a3df97 100644 --- a/platform/linux-dpdk/odp_schedule_if.c +++ b/platform/linux-dpdk/odp_schedule_if.c @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,6 +7,8 @@ #include <odp/autoheader_internal.h> +#include <odp/api/plat/schedule_inline_types.h> + #include <odp_schedule_if.h> #include <odp_init_internal.h> #include <odp_debug_internal.h> @@ -18,24 +20,30 @@ #include <stdlib.h> #include <string.h> +/* Enable visibility to inline headers */ +#include <odp/visibility_begin.h> + +const _odp_schedule_api_fn_t *_odp_sched_api; + +int _odp_schedule_configured(void) +{ + return odp_global_rw->schedule_configured; +} + +#include <odp/visibility_end.h> + extern const schedule_fn_t _odp_schedule_sp_fn; -extern const schedule_api_t _odp_schedule_sp_api; +extern const _odp_schedule_api_fn_t _odp_schedule_sp_api; extern const schedule_fn_t _odp_schedule_basic_fn; -extern const schedule_api_t _odp_schedule_basic_api; +extern const _odp_schedule_api_fn_t _odp_schedule_basic_api; extern const schedule_fn_t _odp_schedule_eventdev_fn; -extern const schedule_api_t _odp_schedule_eventdev_api; +extern const _odp_schedule_api_fn_t _odp_schedule_eventdev_api; const schedule_fn_t *_odp_sched_fn; -const schedule_api_t *_odp_sched_api; int _odp_sched_id; -uint64_t odp_schedule_wait_time(uint64_t ns) -{ - return _odp_sched_api->schedule_wait_time(ns); -} - int odp_schedule_capability(odp_schedule_capability_t *capa) { return _odp_sched_api->schedule_capability(capa); @@ -54,7 +62,7 @@ int odp_schedule_config(const odp_schedule_config_t *config) odp_schedule_config_t defconfig; if (odp_global_rw->schedule_configured) { - ODP_ERR("Scheduler has been configured already\n"); + _ODP_ERR("Scheduler has been configured already\n"); return -1; } @@ -71,56 +79,6 @@ int odp_schedule_config(const odp_schedule_config_t *config) return ret; } -odp_event_t odp_schedule(odp_queue_t *from, uint64_t wait) -{ - ODP_ASSERT(odp_global_rw->schedule_configured); - - return _odp_sched_api->schedule(from, wait); -} - -int odp_schedule_multi(odp_queue_t *from, uint64_t wait, odp_event_t events[], - int num) -{ - ODP_ASSERT(odp_global_rw->schedule_configured); - - return _odp_sched_api->schedule_multi(from, wait, events, num); -} - -int odp_schedule_multi_wait(odp_queue_t *from, odp_event_t events[], int num) -{ - return _odp_sched_api->schedule_multi_wait(from, events, num); -} - -int odp_schedule_multi_no_wait(odp_queue_t *from, odp_event_t events[], int num) -{ - return _odp_sched_api->schedule_multi_no_wait(from, events, num); -} - -void odp_schedule_pause(void) -{ - _odp_sched_api->schedule_pause(); -} - -void odp_schedule_resume(void) -{ - _odp_sched_api->schedule_resume(); -} - -void odp_schedule_release_atomic(void) -{ - _odp_sched_api->schedule_release_atomic(); -} - -void odp_schedule_release_ordered(void) -{ - _odp_sched_api->schedule_release_ordered(); -} - -void odp_schedule_prefetch(int num) -{ - _odp_sched_api->schedule_prefetch(num); -} - int odp_schedule_min_prio(void) { return _odp_sched_api->schedule_min_prio(); @@ -181,36 +139,6 @@ int odp_schedule_group_info(odp_schedule_group_t group, return _odp_sched_api->schedule_group_info(group, info); } -void odp_schedule_order_lock(uint32_t lock_index) -{ - _odp_sched_api->schedule_order_lock(lock_index); -} - -void odp_schedule_order_unlock(uint32_t lock_index) -{ - _odp_sched_api->schedule_order_unlock(lock_index); -} - -void odp_schedule_order_unlock_lock(uint32_t unlock_index, uint32_t lock_index) -{ - _odp_sched_api->schedule_order_unlock_lock(unlock_index, lock_index); -} - -void odp_schedule_order_lock_start(uint32_t lock_index) -{ - _odp_sched_api->schedule_order_lock_start(lock_index); -} - -void odp_schedule_order_lock_wait(uint32_t lock_index) -{ - _odp_sched_api->schedule_order_lock_wait(lock_index); -} - -void odp_schedule_order_wait(void) -{ - _odp_sched_api->schedule_order_wait(); -} - void odp_schedule_print(void) { _odp_sched_api->schedule_print(); @@ -223,7 +151,7 @@ int _odp_schedule_init_global(void) if (sched == NULL || !strcmp(sched, "default")) sched = _ODP_SCHEDULE_DEFAULT; - ODP_PRINT("Using scheduler '%s'\n", sched); + _ODP_PRINT("Using scheduler '%s'\n", sched); if (!strcmp(sched, "basic")) { _odp_sched_id = _ODP_SCHED_ID_BASIC; @@ -238,7 +166,7 @@ int _odp_schedule_init_global(void) _odp_sched_fn = &_odp_schedule_eventdev_fn; _odp_sched_api = &_odp_schedule_eventdev_api; } else { - ODP_ABORT("Unknown scheduler specified via ODP_SCHEDULER\n"); + _ODP_ABORT("Unknown scheduler specified via ODP_SCHEDULER\n"); return -1; } diff --git a/platform/linux-dpdk/odp_shared_memory.c b/platform/linux-dpdk/odp_shared_memory.c index 9e2151a0d..dba91eede 100644 --- a/platform/linux-dpdk/odp_shared_memory.c +++ b/platform/linux-dpdk/odp_shared_memory.c @@ -9,12 +9,14 @@ #include <odp/api/debug.h> #include <odp/api/deprecated.h> -#include <odp/api/plat/strong_types.h> #include <odp/api/shared_memory.h> #include <odp/api/spinlock.h> +#include <odp/api/plat/strong_types.h> + #include <odp_config_internal.h> #include <odp_debug_internal.h> +#include <odp_global_data.h> #include <odp_macros_internal.h> #include <odp_shm_internal.h> @@ -25,6 +27,7 @@ #include <inttypes.h> #include <rte_config.h> +#include <rte_errno.h> #include <rte_lcore.h> #include <rte_memzone.h> @@ -160,8 +163,7 @@ static inline odp_bool_t handle_is_valid(odp_shm_t shm) if (idx < 0 || idx >= SHM_MAX_NB_BLOCKS || shm_tbl->block[idx].mz == NULL) { - ODP_ERR("Invalid odp_shm_t handle: %" PRIu64 "\n", - odp_shm_to_u64(shm)); + _ODP_ERR("Invalid odp_shm_t handle: %" PRIu64 "\n", odp_shm_to_u64(shm)); return 0; } return 1; @@ -173,8 +175,7 @@ int _odp_shm_init_global(const odp_init_t *init ODP_UNUSED) if ((getpid() != odp_global_ro.main_pid) || (syscall(SYS_gettid) != getpid())) { - ODP_ERR("shm_init_global() must be performed by the main " - "ODP process!\n."); + _ODP_ERR("shm_init_global() must be performed by the main ODP process!\n."); return -1; } @@ -182,7 +183,7 @@ int _odp_shm_init_global(const odp_init_t *init ODP_UNUSED) addr = mmap(NULL, sizeof(shm_table_t), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); if (addr == MAP_FAILED) { - ODP_ERR("Unable to mmap the shm block table\n"); + _ODP_ERR("Unable to mmap the shm block table\n"); return -1; } @@ -206,8 +207,7 @@ int _odp_shm_term_global(void) if ((getpid() != odp_global_ro.main_pid) || (syscall(SYS_gettid) != getpid())) { - ODP_ERR("shm_term_global() must be performed by the main " - "ODP process!\n."); + _ODP_ERR("shm_term_global() must be performed by the main ODP process!\n."); return -1; } @@ -215,14 +215,13 @@ int _odp_shm_term_global(void) for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) { block = &shm_tbl->block[idx]; if (block->mz) { - ODP_ERR("block '%s' was never freed (cleaning up...)\n", - block->name); + _ODP_ERR("block '%s' was never freed (cleaning up...)\n", block->name); rte_memzone_free(block->mz); } } /* Free the shared memory block table */ if (munmap(shm_tbl, sizeof(shm_table_t)) < 0) { - ODP_ERR("Unable to munmap the shm block table\n"); + _ODP_ERR("Unable to munmap the shm block table\n"); return -1; } return 0; @@ -256,12 +255,12 @@ odp_shm_t odp_shm_reserve(const char *name, uint64_t size, uint64_t align, uint32_t supported_flgs = SUPPORTED_SHM_FLAGS; if (flags & ~supported_flgs) { - ODP_ERR("Unsupported SHM flag\n"); + _ODP_ERR("Unsupported SHM flag: %" PRIx32 "\n", flags); return ODP_SHM_INVALID; } if (align > SHM_MAX_ALIGN) { - ODP_ERR("Align too large: %" PRIu64 "\n", align); + _ODP_ERR("Align too large: %" PRIu64 "\n", align); return ODP_SHM_INVALID; } @@ -274,7 +273,7 @@ odp_shm_t odp_shm_reserve(const char *name, uint64_t size, uint64_t align, idx = find_free_block(); if (idx < 0) { odp_spinlock_unlock(&shm_tbl->lock); - ODP_ERR("No free SHM blocks left\n"); + _ODP_ERR("No free SHM blocks left\n"); return ODP_SHM_INVALID; } block = &shm_tbl->block[idx]; @@ -286,7 +285,8 @@ odp_shm_t odp_shm_reserve(const char *name, uint64_t size, uint64_t align, rte_socket_id(), mz_flags, align); if (mz == NULL) { odp_spinlock_unlock(&shm_tbl->lock); - ODP_ERR("Reserving DPDK memzone failed\n"); + _ODP_ERR("Reserving DPDK memzone '%s' failed: %s\n", mz_name, + rte_strerror(rte_errno)); return ODP_SHM_INVALID; } @@ -317,12 +317,12 @@ odp_shm_t odp_shm_import(const char *remote_name, odp_instance_t odp_inst, mz = rte_memzone_lookup(mz_name); if (mz == NULL) { - ODP_ERR("Unable to find remote SHM block: %s\n", remote_name); + _ODP_ERR("Unable to find remote SHM block: %s\n", remote_name); return ODP_SHM_INVALID; } if (!(shm_zone(mz)->flags & ODP_SHM_EXPORT)) { - ODP_ERR("Not exported SHM block!\n"); + _ODP_ERR("Not exported SHM block!\n"); return ODP_SHM_INVALID; } @@ -331,7 +331,7 @@ odp_shm_t odp_shm_import(const char *remote_name, odp_instance_t odp_inst, idx = find_free_block(); if (idx < 0) { odp_spinlock_unlock(&shm_tbl->lock); - ODP_ERR("No free SHM blocks left\n"); + _ODP_ERR("No free SHM blocks left\n"); return ODP_SHM_INVALID; } block = &shm_tbl->block[idx]; @@ -443,15 +443,15 @@ void odp_shm_print_all(void) odp_spinlock_lock(&shm_tbl->lock); - ODP_PRINT("\nShared memory blocks\n--------------------\n"); + _ODP_PRINT("\nShared memory blocks\n--------------------\n"); for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) { block = &shm_tbl->block[idx]; if (block->mz == NULL) continue; - ODP_PRINT(" %s: addr: %p, len: %" PRIu64 " page size: " - "%" PRIu64 "\n", block->name, block->mz->addr, - shm_size(block->mz), block->mz->hugepage_sz); + _ODP_PRINT(" %s: addr: %p, len: %" PRIu64 " page size: %" PRIu64 "\n", + block->name, block->mz->addr, + shm_size(block->mz), block->mz->hugepage_sz); } odp_spinlock_unlock(&shm_tbl->lock); @@ -471,16 +471,15 @@ void odp_shm_print(odp_shm_t shm) block = &shm_tbl->block[idx]; - ODP_PRINT("\nSHM block info\n--------------\n"); - ODP_PRINT(" name: %s\n", block->name); - ODP_PRINT(" type: %s\n", block->type == SHM_TYPE_LOCAL ? "local" - : "remote"); - ODP_PRINT(" flags: 0x%x\n", shm_zone(block->mz)->flags); - ODP_PRINT(" start: %p\n", block->mz->addr); - ODP_PRINT(" len: %" PRIu64 "\n", shm_size(block->mz)); - ODP_PRINT(" page size: %" PRIu64 "\n", block->mz->hugepage_sz); - ODP_PRINT(" NUMA ID: %" PRIi32 "\n", block->mz->socket_id); - ODP_PRINT("\n"); + _ODP_PRINT("\nSHM block info\n--------------\n"); + _ODP_PRINT(" name: %s\n", block->name); + _ODP_PRINT(" type: %s\n", block->type == SHM_TYPE_LOCAL ? "local" : "remote"); + _ODP_PRINT(" flags: 0x%x\n", shm_zone(block->mz)->flags); + _ODP_PRINT(" start: %p\n", block->mz->addr); + _ODP_PRINT(" len: %" PRIu64 "\n", shm_size(block->mz)); + _ODP_PRINT(" page size: %" PRIu64 "\n", block->mz->hugepage_sz); + _ODP_PRINT(" NUMA ID: %" PRIi32 "\n", block->mz->socket_id); + _ODP_PRINT("\n"); odp_spinlock_unlock(&shm_tbl->lock); } diff --git a/platform/linux-dpdk/odp_system_info.c b/platform/linux-dpdk/odp_system_info.c index 5f0ddc9f9..c76820e5d 100644 --- a/platform/linux-dpdk/odp_system_info.c +++ b/platform/linux-dpdk/odp_system_info.c @@ -59,8 +59,8 @@ static int read_cache_line_size(void) file = fopen(CACHE_LNSZ_FILE, "rt"); if (file == NULL) { /* File not found */ - ODP_PRINT("WARN: unable to read host CPU cache line size. " - "Using ODP_CACHE_LINE_SIZE instead.\n"); + _ODP_PRINT("WARN: unable to read host CPU cache line size. " + "Using ODP_CACHE_LINE_SIZE instead.\n"); return ODP_CACHE_LINE_SIZE; } @@ -87,13 +87,13 @@ static uint64_t default_huge_page_size(void) while (fgets(str, sizeof(str), file) != NULL) { if (sscanf(str, "Hugepagesize: %8lu kB", &sz) == 1) { - ODP_DBG("default hp size is %lu kB\n", sz); + _ODP_DBG("default hp size is %lu kB\n", sz); fclose(file); return (uint64_t)sz * 1024; } } - ODP_ERR("unable to get default hp size\n"); + _ODP_ERR("unable to get default hp size\n"); fclose(file); return 0; } @@ -141,7 +141,7 @@ static char *get_hugepage_dir(uint64_t hugepage_sz) while (fgets(buf, sizeof(buf), fd)) { if (rte_strsplit(buf, sizeof(buf), tokens, _FIELDNAME_MAX, split_tok) != _FIELDNAME_MAX) { - ODP_ERR("Error parsing %s\n", proc_mounts); + _ODP_ERR("Error parsing %s\n", proc_mounts); break; /* return NULL */ } @@ -218,14 +218,14 @@ static int system_cache_line(system_info_t *sysinfo) ret = read_cache_line_size(); if (ret == 0) { - ODP_ERR("read_cache_line_size failed.\n"); + _ODP_ERR("read_cache_line_size failed.\n"); return -1; } sysinfo->cache_line_size = ret; if (ret != ODP_CACHE_LINE_SIZE) - ODP_PRINT("WARN: host CPU cache line size and ODP_CACHE_LINE_SIZE don't match.\n"); + _ODP_PRINT("WARN: host CPU cache line size and ODP_CACHE_LINE_SIZE don't match.\n"); return 0; } @@ -250,21 +250,21 @@ static int read_config_file(void) str = "system.cpu_mhz"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } odp_global_ro.system_info.default_cpu_hz = (uint64_t)val * 1000000; str = "system.cpu_mhz_max"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } odp_global_ro.system_info.default_cpu_hz_max = (uint64_t)val * 1000000; str = "system.cpu_hz_static"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } odp_global_ro.system_info.cpu_hz_static = !!val; @@ -274,17 +274,17 @@ static int read_config_file(void) static void print_compiler_info(void) { - ODP_PRINT("Compiler defines:\n"); - ODP_PRINT(" __GCC_ATOMIC_LLONG_LOCK_FREE: %d\n", __GCC_ATOMIC_LLONG_LOCK_FREE); - ODP_PRINT(" __GCC_ATOMIC_LONG_LOCK_FREE: %d\n", __GCC_ATOMIC_LONG_LOCK_FREE); - ODP_PRINT(" __GCC_ATOMIC_INT_LOCK_FREE: %d\n", __GCC_ATOMIC_INT_LOCK_FREE); - ODP_PRINT(" __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16: "); + _ODP_PRINT("Compiler defines:\n"); + _ODP_PRINT(" __GCC_ATOMIC_LLONG_LOCK_FREE: %d\n", __GCC_ATOMIC_LLONG_LOCK_FREE); + _ODP_PRINT(" __GCC_ATOMIC_LONG_LOCK_FREE: %d\n", __GCC_ATOMIC_LONG_LOCK_FREE); + _ODP_PRINT(" __GCC_ATOMIC_INT_LOCK_FREE: %d\n", __GCC_ATOMIC_INT_LOCK_FREE); + _ODP_PRINT(" __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16: "); #ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 - ODP_PRINT("1\n"); + _ODP_PRINT("1\n"); #else - ODP_PRINT("0\n"); + _ODP_PRINT("0\n"); #endif - ODP_PRINT("\n"); + _ODP_PRINT("\n"); } /* @@ -307,9 +307,9 @@ int _odp_system_info_init(void) /* Check that CONFIG_NUM_CPU_IDS is large enough */ num_cpus = get_nprocs_conf(); if (num_cpus > CONFIG_NUM_CPU_IDS) - ODP_ERR("Unable to handle all %d " - "CPU IDs. Increase CONFIG_NUM_CPU_IDS value.\n", - num_cpus); + _ODP_ERR("Unable to handle all %d " + "CPU IDs. Increase CONFIG_NUM_CPU_IDS value.\n", + num_cpus); /* Read and save all CPU frequencies for static mode */ for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) @@ -370,7 +370,7 @@ uint64_t odp_cpu_hz(void) uint64_t odp_cpu_hz_id(int id) { - ODP_ASSERT(id >= 0 && id < CONFIG_NUM_CPU_IDS); + _ODP_ASSERT(id >= 0 && id < CONFIG_NUM_CPU_IDS); if (odp_global_ro.system_info.cpu_hz_static) return cpu_hz_static(id); @@ -417,8 +417,7 @@ int odp_sys_huge_page_size_all(uint64_t size[], int num) /* See: kernel.org: hugetlbpage.txt */ dir = opendir("/sys/kernel/mm/hugepages"); if (!dir) { - ODP_PRINT("Failed to open /sys/kernel/mm/hugepages: %s\n", - strerror(errno)); + _ODP_PRINT("Failed to open /sys/kernel/mm/hugepages: %s\n", strerror(errno)); return 0; } @@ -516,7 +515,7 @@ void odp_sys_info_print(void) num_cpu, cpumask_str); str[len] = '\0'; - ODP_PRINT("%s", str); + _ODP_PRINT("%s", str); _odp_sys_info_print_arch(); } @@ -525,26 +524,26 @@ void odp_sys_config_print(void) { /* Print ODP_CONFIG_FILE default and override values */ if (_odp_libconfig_print()) - ODP_ERR("Config file print failed\n"); - - ODP_PRINT("\n\nodp_config_internal.h values:\n" - "-----------------------------\n"); - ODP_PRINT("CONFIG_NUM_CPU_IDS: %i\n", CONFIG_NUM_CPU_IDS); - ODP_PRINT("ODP_CONFIG_POOLS: %i\n", ODP_CONFIG_POOLS); - ODP_PRINT("CONFIG_INTERNAL_QUEUES: %i\n", CONFIG_INTERNAL_QUEUES); - ODP_PRINT("CONFIG_MAX_PLAIN_QUEUES: %i\n", CONFIG_MAX_PLAIN_QUEUES); - ODP_PRINT("CONFIG_MAX_SCHED_QUEUES: %i\n", CONFIG_MAX_SCHED_QUEUES); - ODP_PRINT("CONFIG_MAX_QUEUES: %i\n", CONFIG_MAX_QUEUES); - ODP_PRINT("CONFIG_QUEUE_MAX_ORD_LOCKS: %i\n", CONFIG_QUEUE_MAX_ORD_LOCKS); - ODP_PRINT("ODP_CONFIG_PKTIO_ENTRIES: %i\n", ODP_CONFIG_PKTIO_ENTRIES); - ODP_PRINT("ODP_CONFIG_BUFFER_ALIGN_MIN: %i\n", ODP_CONFIG_BUFFER_ALIGN_MIN); - ODP_PRINT("ODP_CONFIG_BUFFER_ALIGN_MAX: %i\n", ODP_CONFIG_BUFFER_ALIGN_MAX); - ODP_PRINT("CONFIG_PACKET_TAILROOM: %i\n", CONFIG_PACKET_TAILROOM); - ODP_PRINT("CONFIG_PACKET_SEG_SIZE: %i\n", CONFIG_PACKET_SEG_SIZE); - ODP_PRINT("CONFIG_PACKET_SEG_LEN_MIN: %i\n", CONFIG_PACKET_SEG_LEN_MIN); - ODP_PRINT("CONFIG_PACKET_MAX_SEG_LEN: %i\n", CONFIG_PACKET_MAX_SEG_LEN); - ODP_PRINT("CONFIG_SHM_BLOCKS: %i\n", CONFIG_SHM_BLOCKS); - ODP_PRINT("CONFIG_BURST_SIZE: %i\n", CONFIG_BURST_SIZE); - ODP_PRINT("CONFIG_POOL_MAX_NUM: %i\n", CONFIG_POOL_MAX_NUM); - ODP_PRINT("\n"); + _ODP_ERR("Config file print failed\n"); + + _ODP_PRINT("\n\nodp_config_internal.h values:\n" + "-----------------------------\n"); + _ODP_PRINT("CONFIG_NUM_CPU_IDS: %i\n", CONFIG_NUM_CPU_IDS); + _ODP_PRINT("ODP_CONFIG_POOLS: %i\n", ODP_CONFIG_POOLS); + _ODP_PRINT("CONFIG_INTERNAL_QUEUES: %i\n", CONFIG_INTERNAL_QUEUES); + _ODP_PRINT("CONFIG_MAX_PLAIN_QUEUES: %i\n", CONFIG_MAX_PLAIN_QUEUES); + _ODP_PRINT("CONFIG_MAX_SCHED_QUEUES: %i\n", CONFIG_MAX_SCHED_QUEUES); + _ODP_PRINT("CONFIG_MAX_QUEUES: %i\n", CONFIG_MAX_QUEUES); + _ODP_PRINT("CONFIG_QUEUE_MAX_ORD_LOCKS: %i\n", CONFIG_QUEUE_MAX_ORD_LOCKS); + _ODP_PRINT("ODP_CONFIG_PKTIO_ENTRIES: %i\n", ODP_CONFIG_PKTIO_ENTRIES); + _ODP_PRINT("ODP_CONFIG_BUFFER_ALIGN_MIN: %i\n", ODP_CONFIG_BUFFER_ALIGN_MIN); + _ODP_PRINT("ODP_CONFIG_BUFFER_ALIGN_MAX: %i\n", ODP_CONFIG_BUFFER_ALIGN_MAX); + _ODP_PRINT("CONFIG_PACKET_TAILROOM: %i\n", CONFIG_PACKET_TAILROOM); + _ODP_PRINT("CONFIG_PACKET_SEG_SIZE: %i\n", CONFIG_PACKET_SEG_SIZE); + _ODP_PRINT("CONFIG_PACKET_SEG_LEN_MIN: %i\n", CONFIG_PACKET_SEG_LEN_MIN); + _ODP_PRINT("CONFIG_PACKET_MAX_SEG_LEN: %i\n", CONFIG_PACKET_MAX_SEG_LEN); + _ODP_PRINT("CONFIG_SHM_BLOCKS: %i\n", CONFIG_SHM_BLOCKS); + _ODP_PRINT("CONFIG_BURST_SIZE: %i\n", CONFIG_BURST_SIZE); + _ODP_PRINT("CONFIG_POOL_MAX_NUM: %i\n", CONFIG_POOL_MAX_NUM); + _ODP_PRINT("\n"); } diff --git a/platform/linux-dpdk/odp_thread.c b/platform/linux-dpdk/odp_thread.c index 7ab41cf72..9f30aabb0 100644 --- a/platform/linux-dpdk/odp_thread.c +++ b/platform/linux-dpdk/odp_thread.c @@ -66,11 +66,11 @@ int _odp_thread_init_global(void) const char *str = "system.thread_count_max"; if (!_odp_libconfig_lookup_int(str, &num_max)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } if (num_max <= 0) { - ODP_ERR("Config option '%s' not valid.\n", str); + _ODP_ERR("Config option '%s' not valid.\n", str); return -1; } if (num_max > ODP_THREAD_COUNT_MAX) @@ -92,8 +92,8 @@ int _odp_thread_init_global(void) odp_spinlock_init(&thread_globals->lock); odp_atomic_init_u64(&thread_globals->thrmask_all_epoch, 0); thread_globals->num_max = num_max; - ODP_PRINT("System config:\n"); - ODP_PRINT(" system.thread_count_max: %d\n\n", num_max); + _ODP_PRINT("System config:\n"); + _ODP_PRINT(" system.thread_count_max: %d\n\n", num_max); return 0; } @@ -107,11 +107,11 @@ int _odp_thread_term_global(void) odp_spinlock_unlock(&thread_globals->lock); if (num) - ODP_ERR("%u threads have not called odp_term_local().\n", num); + _ODP_ERR("%u threads have not called odp_term_local().\n", num); ret = odp_shm_free(thread_globals->shm); if (ret < 0) - ODP_ERR("shm free failed for odp_thread_globals"); + _ODP_ERR("shm free failed for odp_thread_globals"); return ret; } @@ -213,14 +213,14 @@ int _odp_thread_init_local(odp_thread_type_t type) odp_spinlock_unlock(&thread_globals->lock); if (id < 0) { - ODP_ERR("Too many threads\n"); + _ODP_ERR("Too many threads\n"); return -1; } cpu = sched_getcpu(); if (cpu < 0) { - ODP_ERR("getcpu failed\n"); + _ODP_ERR("getcpu failed\n"); return -1; } @@ -279,7 +279,7 @@ int _odp_thread_term_local(void) odp_spinlock_unlock(&thread_globals->lock); if (num < 0) { - ODP_ERR("failed to free thread id %i", id); + _ODP_ERR("failed to free thread id %i", id); return -1; } diff --git a/platform/linux-dpdk/odp_time.c b/platform/linux-dpdk/odp_time.c index e7cd5d1bb..fbbc9358d 100644 --- a/platform/linux-dpdk/odp_time.c +++ b/platform/linux-dpdk/odp_time.c @@ -62,7 +62,7 @@ odp_time_t _odp_timespec_cur(void) ret = clock_gettime(CLOCK_MONOTONIC_RAW, &sys_time); if (odp_unlikely(ret != 0)) - ODP_ABORT("clock_gettime failed\n"); + _ODP_ABORT("clock_gettime failed\n"); time.nsec = time_spec_diff_nsec(&sys_time, start_time); @@ -78,7 +78,7 @@ static inline uint64_t time_spec_res(void) ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); if (odp_unlikely(ret != 0)) - ODP_ABORT("clock_getres failed\n"); + _ODP_ABORT("clock_getres failed\n"); return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; } @@ -284,8 +284,7 @@ int _odp_time_init_global(void) if (global->hw_freq_hz == 0) return -1; - ODP_PRINT("HW time counter freq: %" PRIu64 " hz\n\n", - global->hw_freq_hz); + _ODP_PRINT("HW time counter freq: %" PRIu64 " hz\n\n", global->hw_freq_hz); global->hw_start = _odp_cpu_global_time(); return 0; diff --git a/platform/linux-dpdk/odp_timer.c b/platform/linux-dpdk/odp_timer.c index 7e59775d3..5296a9d3c 100644 --- a/platform/linux-dpdk/odp_timer.c +++ b/platform/linux-dpdk/odp_timer.c @@ -25,6 +25,7 @@ #include <odp_libconfig_internal.h> #include <odp_macros_internal.h> #include <odp_pool_internal.h> +#include <odp_print_internal.h> #include <odp_queue_if.h> #include <odp_ring_u32_internal.h> #include <odp_thread_internal.h> @@ -198,7 +199,7 @@ static void timer_cb(struct rte_timer *rte_timer, void *arg ODP_UNUSED) odp_ticketlock_lock(&timer->lock); if (timer->state != TICKING) { - ODP_ERR("Timer has been cancelled or freed.\n"); + _ODP_ERR("Timer has been cancelled or freed.\n"); odp_ticketlock_unlock(&timer->lock); return; } @@ -213,7 +214,7 @@ static void timer_cb(struct rte_timer *rte_timer, void *arg ODP_UNUSED) odp_ticketlock_unlock(&timer->lock); if (odp_unlikely(odp_queue_enq(queue, event))) { - ODP_ERR("Timeout event enqueue failed.\n"); + _ODP_ERR("Timeout event enqueue failed.\n"); odp_event_free(event); } } @@ -320,8 +321,7 @@ int _odp_timer_init_global(const odp_init_t *params) ODP_CACHE_LINE_SIZE, 0); if (shm == ODP_SHM_INVALID) { - ODP_ERR("Global data alloc (%zu bytes) failed\n", - sizeof(timer_global_t)); + _ODP_ERR("Global data alloc (%zu bytes) failed\n", sizeof(timer_global_t)); return -1; } @@ -331,48 +331,48 @@ int _odp_timer_init_global(const odp_init_t *params) timer_global->shm = shm; odp_ticketlock_init(&timer_global->lock); - ODP_PRINT("\nTimer config:\n"); + _ODP_PRINT("\nTimer config:\n"); conf_str = "timer.inline_poll_interval"; if (!_odp_libconfig_lookup_int(conf_str, &val)) { - ODP_ERR("Config option '%s' not found.\n", conf_str); + _ODP_ERR("Config option '%s' not found.\n", conf_str); odp_shm_free(shm); return -1; } timer_global->poll_interval = val; - ODP_PRINT(" %s: %d\n", conf_str, val); + _ODP_PRINT(" %s: %d\n", conf_str, val); conf_str = "timer.inline_poll_interval_nsec"; if (!_odp_libconfig_lookup_int(conf_str, &val)) { - ODP_ERR("Config option '%s' not found.\n", conf_str); + _ODP_ERR("Config option '%s' not found.\n", conf_str); odp_shm_free(shm); return -1; } timer_global->poll_interval_nsec = val; timer_global->poll_interval_time = odp_time_global_from_ns(timer_global->poll_interval_nsec); - ODP_PRINT(" %s: %d\n", conf_str, val); + _ODP_PRINT(" %s: %d\n", conf_str, val); conf_str = "timer.alternate"; if (!_odp_libconfig_lookup_int(conf_str, &val)) { - ODP_ERR("Config option '%s' not found.\n", conf_str); + _ODP_ERR("Config option '%s' not found.\n", conf_str); odp_shm_free(shm); return -1; } timer_global->use_alternate = !!val; - ODP_PRINT(" %s: %" PRIu8 "\n", conf_str, timer_global->use_alternate); + _ODP_PRINT(" %s: %" PRIu8 "\n", conf_str, timer_global->use_alternate); - ODP_PRINT("\n"); + _ODP_PRINT("\n"); if (rte_timer_subsystem_init()) { - ODP_ERR("Initializing DPDK timer library failed\n"); + _ODP_ERR("Initializing DPDK timer library failed\n"); odp_shm_free(shm); return -1; } if (timer_global->use_alternate) { if (rte_timer_data_alloc(&timer_global->data_id)) { - ODP_ERR("Failed to allocate DPDK timer data instance\n"); + _ODP_ERR("Failed to allocate DPDK timer data instance\n"); odp_shm_free(shm); return -1; } @@ -392,14 +392,14 @@ int _odp_timer_term_global(void) { if (timer_global && timer_global->use_alternate) { if (rte_timer_data_dealloc(timer_global->data_id)) { - ODP_ERR("Failed to deallocate DPDK timer data instance\n"); + _ODP_ERR("Failed to deallocate DPDK timer data instance\n"); return -1; } } rte_timer_subsystem_finalize(); if (timer_global && odp_shm_free(timer_global->shm)) { - ODP_ERR("Shm free failed for odp_timer\n"); + _ODP_ERR("Shm free failed for odp_timer\n"); return -1; } @@ -447,7 +447,7 @@ void _odp_timer_run_inline(int dec) /* Check timer pools */ ret = timer_global->ops.manage(); if (odp_unlikely(ret)) - ODP_ERR("RTE timer manage failed: %d\n", ret); + _ODP_ERR("RTE timer manage failed: %d\n", ret); } static inline uint64_t tmo_ticks_to_ns_round_up(uint64_t tmo_ticks) @@ -468,7 +468,7 @@ int odp_timer_capability(odp_timer_clk_src_t clk_src, uint64_t min_tmo = tmo_ticks_to_ns_round_up(MIN_TMO_CYCLES); if (clk_src != ODP_CLOCK_DEFAULT) { - ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", clk_src); + _ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", clk_src); return -1; } @@ -503,12 +503,12 @@ int odp_timer_res_capability(odp_timer_clk_src_t clk_src, uint64_t min_tmo = tmo_ticks_to_ns_round_up(MIN_TMO_CYCLES); if (clk_src != ODP_CLOCK_DEFAULT) { - ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", clk_src); + _ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", clk_src); return -1; } if (res_capa->min_tmo) { - ODP_ERR("Only res_ns or max_tmo based quaries supported\n"); + _ODP_ERR("Only res_ns or max_tmo based quaries supported\n"); return -1; } @@ -531,14 +531,14 @@ int odp_timer_periodic_capability(odp_timer_clk_src_t clk_src, uint64_t multiplier; if (clk_src != ODP_CLOCK_DEFAULT) { - ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", clk_src); + _ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", clk_src); return -1; } freq = odp_fract_u64_to_dbl(&capa->base_freq_hz); if (freq < MIN_BASE_HZ || freq > MAX_BASE_HZ) { - ODP_ERR("Base frequency not supported (min: %f, max %f)\n", - (double)MIN_BASE_HZ, (double)MAX_BASE_HZ); + _ODP_ERR("Base frequency not supported (min: %f, max %f)\n", + (double)MIN_BASE_HZ, (double)MAX_BASE_HZ); return -1; } @@ -578,39 +578,39 @@ odp_timer_pool_t odp_timer_pool_create(const char *name, int periodic = (param->timer_type == ODP_TIMER_TYPE_PERIODIC) ? 1 : 0; if (odp_global_ro.init_param.not_used.feat.timer) { - ODP_ERR("Trying to use disabled ODP feature.\n"); + _ODP_ERR("Trying to use disabled ODP feature.\n"); return ODP_TIMER_POOL_INVALID; } if (param->clk_src != ODP_CLOCK_DEFAULT) { - ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", param->clk_src); + _ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", param->clk_src); return ODP_TIMER_POOL_INVALID; } if (param->timer_type != ODP_TIMER_TYPE_SINGLE && param->timer_type != ODP_TIMER_TYPE_PERIODIC) { - ODP_ERR("Bad timer type %i\n", param->timer_type); + _ODP_ERR("Bad timer type %i\n", param->timer_type); return ODP_TIMER_POOL_INVALID; } if ((param->res_ns && param->res_hz) || (param->res_ns == 0 && param->res_hz == 0)) { - ODP_ERR("Invalid timeout resolution\n"); + _ODP_ERR("Invalid timeout resolution\n"); return ODP_TIMER_POOL_INVALID; } if (param->res_hz == 0 && param->res_ns < MAX_RES_NS) { - ODP_ERR("Too high resolution\n"); + _ODP_ERR("Too high resolution\n"); return ODP_TIMER_POOL_INVALID; } if (param->res_ns == 0 && param->res_hz > MAX_RES_HZ) { - ODP_ERR("Too high resolution\n"); + _ODP_ERR("Too high resolution\n"); return ODP_TIMER_POOL_INVALID; } if (param->num_timers > MAX_TIMERS) { - ODP_ERR("Too many timers\n"); + _ODP_ERR("Too many timers\n"); return ODP_TIMER_POOL_INVALID; } @@ -628,14 +628,14 @@ odp_timer_pool_t odp_timer_pool_create(const char *name, max_multiplier = param->periodic.max_multiplier; if (base_freq < MIN_BASE_HZ || base_freq > MAX_BASE_HZ) { - ODP_ERR("Bad base frequency: %f\n", base_freq); + _ODP_ERR("Bad base frequency: %f\n", base_freq); return ODP_TIMER_POOL_INVALID; } max_capa = max_multiplier_capa(base_freq); if (max_multiplier == 0 || max_multiplier > max_capa) { - ODP_ERR("Bad max multiplier: %" PRIu64 "\n", max_multiplier); + _ODP_ERR("Bad max multiplier: %" PRIu64 "\n", max_multiplier); return ODP_TIMER_POOL_INVALID; } @@ -660,7 +660,7 @@ odp_timer_pool_t odp_timer_pool_create(const char *name, if (timer_global->num_timer_pools >= MAX_TIMER_POOLS) { odp_ticketlock_unlock(&timer_global->lock); - ODP_DBG("No more free timer pools\n"); + _ODP_DBG("No more free timer pools\n"); return ODP_TIMER_POOL_INVALID; } @@ -793,7 +793,7 @@ int odp_timer_pool_info(odp_timer_pool_t tp, uint64_t freq_hz = rte_get_timer_hz(); if (odp_unlikely(tp == ODP_TIMER_POOL_INVALID)) { - ODP_ERR("Invalid timer pool.\n"); + _ODP_ERR("Invalid timer pool.\n"); return -1; } @@ -831,12 +831,12 @@ odp_timer_t odp_timer_alloc(odp_timer_pool_t tp, timer_pool_t *timer_pool = timer_pool_from_hdl(tp); if (odp_unlikely(tp == ODP_TIMER_POOL_INVALID)) { - ODP_ERR("Invalid timer pool.\n"); + _ODP_ERR("Invalid timer pool.\n"); return ODP_TIMER_INVALID; } if (odp_unlikely(queue == ODP_QUEUE_INVALID)) { - ODP_ERR("%s: Invalid queue handle.\n", timer_pool->name); + _ODP_ERR("%s: Invalid queue handle.\n", timer_pool->name); return ODP_TIMER_INVALID; } @@ -878,7 +878,7 @@ retry: odp_ticketlock_lock(&timer->lock); if (timer->state == TICKING) { - ODP_DBG("Freeing active timer.\n"); + _ODP_DBG("Freeing active timer.\n"); if (timer_global->ops.stop(&timer->rte_timer)) { /* Another core runs timer callback function. */ @@ -939,10 +939,9 @@ retry: } if (rel_tick < MIN_TMO_CYCLES) { - ODP_DBG("Too early\n"); - ODP_DBG(" cur_tick %" PRIu64 ", abs_tick %" PRIu64 "\n", - cur_tick, abs_tick); - ODP_DBG(" num_retry %i\n", num_retry); + _ODP_DBG("Too early\n"); + _ODP_DBG(" cur_tick %" PRIu64 ", abs_tick %" PRIu64 "\n", cur_tick, abs_tick); + _ODP_DBG(" num_retry %i\n", num_retry); return ODP_TIMER_TOO_NEAR; } @@ -1040,7 +1039,7 @@ int odp_timer_start(odp_timer_t timer, const odp_timer_start_t *start_param) /* Check that timer was not active */ if (odp_unlikely(tmo_ev != ODP_EVENT_INVALID)) { - ODP_ERR("Timer was active already\n"); + _ODP_ERR("Timer was active already\n"); odp_event_free(tmo_ev); } @@ -1069,23 +1068,23 @@ int odp_timer_periodic_start(odp_timer_t timer_hdl, int ret; if (odp_unlikely(!tp->periodic)) { - ODP_ERR("Not a periodic timer\n"); + _ODP_ERR("Not a periodic timer\n"); return ODP_TIMER_FAIL; } if (odp_unlikely(multiplier == 0 || multiplier > tp->max_multiplier)) { - ODP_ERR("Bad frequency multiplier: %" PRIu64 "\n", multiplier); + _ODP_ERR("Bad frequency multiplier: %" PRIu64 "\n", multiplier); return ODP_TIMER_FAIL; } if (odp_unlikely(odp_event_type(tmo_ev) != ODP_EVENT_TIMEOUT)) { - ODP_ERR("Event type is not timeout\n"); + _ODP_ERR("Event type is not timeout\n"); return ODP_TIMER_FAIL; } period_ns = (uint64_t)((double)ODP_TIME_SEC_IN_NS / freq); if (period_ns == 0) { - ODP_ERR("Too high periodic timer frequency: %f\n", freq); + _ODP_ERR("Too high periodic timer frequency: %f\n", freq); return ODP_TIMER_FAIL; } @@ -1104,7 +1103,7 @@ int odp_timer_periodic_start(odp_timer_t timer_hdl, /* Check that timer was not active */ if (odp_unlikely(tmo_ev != ODP_EVENT_INVALID)) { - ODP_ERR("Timer was active already\n"); + _ODP_ERR("Timer was active already\n"); odp_event_free(tmo_ev); } @@ -1120,7 +1119,7 @@ int odp_timer_periodic_ack(odp_timer_t timer_hdl, odp_event_t tmo_ev) int ret; if (odp_unlikely(odp_event_type(tmo_ev) != ODP_EVENT_TIMEOUT)) { - ODP_ERR("Event type is not timeout\n"); + _ODP_ERR("Event type is not timeout\n"); return -1; } @@ -1140,12 +1139,12 @@ int odp_timer_periodic_ack(odp_timer_t timer_hdl, odp_event_t tmo_ev) /* Send delayed timeout immediately to catch-up */ if (ret == ODP_TIMER_TOO_NEAR) { if (odp_unlikely(odp_queue_enq(timer->queue, tmo_ev))) { - ODP_ERR("Failed to enqueue catch-up timeout event\n"); + _ODP_ERR("Failed to enqueue catch-up timeout event\n"); return -1; } return 0; } - ODP_ERR("Failed to re-arm periodic timer: %d\n", ret); + _ODP_ERR("Failed to re-arm periodic timer: %d\n", ret); return -1; } @@ -1182,7 +1181,7 @@ int odp_timer_periodic_cancel(odp_timer_t timer_hdl) int ret; if (odp_unlikely(timer_hdl == ODP_TIMER_INVALID)) { - ODP_ERR("Bad timer handle\n"); + _ODP_ERR("Bad timer handle\n"); return -1; } @@ -1191,7 +1190,7 @@ int odp_timer_periodic_cancel(odp_timer_t timer_hdl) event = timer->tmo_event; if (odp_unlikely(!tp->periodic)) { - ODP_ERR("Not a periodic timer\n"); + _ODP_ERR("Not a periodic timer\n"); return -1; } @@ -1206,7 +1205,7 @@ int odp_timer_periodic_cancel(odp_timer_t timer_hdl) if (ret == 0 && timer->state == TICKING) { timer->state = NOT_TICKING; if (odp_unlikely(odp_queue_enq(timer->queue, event))) { - ODP_ERR("Failed to enqueue final timeout event\n"); + _ODP_ERR("Failed to enqueue final timeout event\n"); _odp_event_free(event); } } @@ -1243,11 +1242,11 @@ odp_timeout_t odp_timeout_alloc(odp_pool_t pool_hdl) odp_event_t event; pool_t *pool; - ODP_ASSERT(pool_hdl != ODP_POOL_INVALID); + _ODP_ASSERT(pool_hdl != ODP_POOL_INVALID); pool = _odp_pool_entry(pool_hdl); - ODP_ASSERT(pool->type == ODP_POOL_TIMEOUT); + _ODP_ASSERT(pool->type == ODP_POOL_TIMEOUT); event = _odp_event_alloc(pool); if (odp_unlikely(event == ODP_EVENT_INVALID)) @@ -1264,74 +1263,105 @@ void odp_timeout_free(odp_timeout_t tmo) void odp_timer_pool_print(odp_timer_pool_t timer_pool) { timer_pool_t *tp; + int len = 0; + int max_len = 512; + int n = max_len - 1; + char str[max_len]; if (timer_pool == ODP_TIMER_POOL_INVALID) { - ODP_ERR("Bad timer pool handle\n"); + _ODP_ERR("Bad timer pool handle\n"); return; } tp = timer_pool_from_hdl(timer_pool); - ODP_PRINT("\nTimer pool info\n"); - ODP_PRINT("---------------\n"); - ODP_PRINT(" timer pool %p\n", (void *)tp); - ODP_PRINT(" name %s\n", tp->name); - ODP_PRINT(" num timers %u\n", tp->cur_timers); - ODP_PRINT(" hwm timers %u\n", tp->hwm_timers); - ODP_PRINT(" num tp %i\n", timer_global->num_timer_pools); - ODP_PRINT(" periodic %" PRIu8 "\n", tp->periodic); - ODP_PRINT("\n"); + len += _odp_snprint(&str[len], n - len, "Timer pool info\n"); + len += _odp_snprint(&str[len], n - len, "---------------\n"); + len += _odp_snprint(&str[len], n - len, " handle 0x%" PRIx64 "\n", + odp_timer_pool_to_u64(timer_pool)); + len += _odp_snprint(&str[len], n - len, " name %s\n", tp->name); + len += _odp_snprint(&str[len], n - len, " num timers %u\n", tp->cur_timers); + len += _odp_snprint(&str[len], n - len, " hwm timers %u\n", tp->hwm_timers); + len += _odp_snprint(&str[len], n - len, " num tp %i\n", + timer_global->num_timer_pools); + len += _odp_snprint(&str[len], n - len, " periodic %" PRIu8 "\n", tp->periodic); + str[len] = 0; + + _ODP_PRINT("%s\n", str); } void odp_timer_print(odp_timer_t timer_hdl) { timer_entry_t *timer = timer_from_hdl(timer_hdl); + int len = 0; + int max_len = 512; + int n = max_len - 1; + char str[max_len]; if (timer_hdl == ODP_TIMER_INVALID) { - ODP_ERR("Bad timer handle\n"); + _ODP_ERR("Bad timer handle\n"); return; } - ODP_PRINT("\nTimer info\n"); - ODP_PRINT("----------\n"); - ODP_PRINT(" timer pool %p\n", (void *)timer->timer_pool); - ODP_PRINT(" timer index %" PRIu32 "\n", timer->timer_idx); - ODP_PRINT(" dest queue 0x%" PRIx64 "\n", odp_queue_to_u64(timer->queue)); - ODP_PRINT(" user ptr %p\n", timer->user_ptr); - ODP_PRINT(" state %s\n", - (timer->state == NOT_TICKING) ? "not ticking" : - (timer->state == EXPIRED ? "expired" : "ticking")); - ODP_PRINT(" periodic ticks %" PRIu64 "\n", timer->periodic_ticks); - ODP_PRINT("\n"); + len += _odp_snprint(&str[len], n - len, "Timer info\n"); + len += _odp_snprint(&str[len], n - len, "----------\n"); + len += _odp_snprint(&str[len], n - len, " handle 0x%" PRIx64 "\n", + odp_timer_to_u64(timer_hdl)); + len += _odp_snprint(&str[len], n - len, " timer pool 0x%" PRIx64 "\n", + odp_timer_pool_to_u64(timer_pool_to_hdl(timer->timer_pool))); + len += _odp_snprint(&str[len], n - len, " timer index %" PRIu32 "\n", timer->timer_idx); + len += _odp_snprint(&str[len], n - len, " dest queue 0x%" PRIx64 "\n", + odp_queue_to_u64(timer->queue)); + len += _odp_snprint(&str[len], n - len, " user ptr %p\n", timer->user_ptr); + len += _odp_snprint(&str[len], n - len, " state %s\n", + (timer->state == NOT_TICKING) ? "not ticking" : + (timer->state == EXPIRED ? "expired" : "ticking")); + len += _odp_snprint(&str[len], n - len, " periodic ticks %" PRIu64 "\n", + timer->periodic_ticks); + str[len] = 0; + + _ODP_PRINT("%s\n", str); } void odp_timeout_print(odp_timeout_t tmo) { const odp_timeout_hdr_t *tmo_hdr; odp_timer_t timer; + int len = 0; + int max_len = 512; + int n = max_len - 1; + char str[max_len]; if (tmo == ODP_TIMEOUT_INVALID) { - ODP_ERR("Bad timeout handle\n"); + _ODP_ERR("Bad timeout handle\n"); return; } tmo_hdr = timeout_to_hdr(tmo); timer = tmo_hdr->timer; - ODP_PRINT("\nTimeout info\n"); - ODP_PRINT("------------\n"); - ODP_PRINT(" tmo handle 0x%" PRIx64 "\n", odp_timeout_to_u64(tmo)); - ODP_PRINT(" expiration %" PRIu64 "\n", tmo_hdr->expiration); - ODP_PRINT(" user ptr %p\n", tmo_hdr->user_ptr); + len += _odp_snprint(&str[len], n - len, "Timeout info\n"); + len += _odp_snprint(&str[len], n - len, "------------\n"); + len += _odp_snprint(&str[len], n - len, " handle 0x%" PRIx64 "\n", + odp_timeout_to_u64(tmo)); + len += _odp_snprint(&str[len], n - len, " expiration %" PRIu64 "\n", + tmo_hdr->expiration); + len += _odp_snprint(&str[len], n - len, " user ptr %p\n", tmo_hdr->user_ptr); + len += _odp_snprint(&str[len], n - len, " user area %p\n", tmo_hdr->uarea_addr); if (timer != ODP_TIMER_INVALID) { timer_entry_t *timer_entry = timer_from_hdl(timer); timer_pool_t *tp = timer_entry->timer_pool; - ODP_PRINT(" timer pool %p\n", (void *)tp); - ODP_PRINT(" timer index %u\n", timer_entry->timer_idx); - ODP_PRINT(" periodic %i\n", tp->periodic); + len += _odp_snprint(&str[len], n - len, " timer pool 0x%" PRIx64 "\n", + odp_timer_pool_to_u64(timer_pool_to_hdl(tp))); + len += _odp_snprint(&str[len], n - len, " timer 0x%" PRIx64 "\n", + odp_timer_to_u64(timer)); + len += _odp_snprint(&str[len], n - len, " timer index %u\n", + timer_entry->timer_idx); + len += _odp_snprint(&str[len], n - len, " periodic %i\n", tp->periodic); } + str[len] = 0; - ODP_PRINT("\n"); + _ODP_PRINT("%s\n", str); } diff --git a/platform/linux-dpdk/test/Makefile.am b/platform/linux-dpdk/test/Makefile.am index 1496c0558..5758a2af7 100644 --- a/platform/linux-dpdk/test/Makefile.am +++ b/platform/linux-dpdk/test/Makefile.am @@ -38,8 +38,3 @@ $(TESTENV): echo "TESTS=\"$(TESTS)\"" > $@ echo "$(TESTS_ENVIRONMENT)" >> $@ echo "$(LOG_COMPILER)" >> $@ - -if test_installdir -installcheck-local: - $(DESTDIR)/$(testdir)/run-test.sh $(TESTNAME) -endif diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 708526558..ce8b0ba09 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -31,8 +31,10 @@ odpapiplatincludedir= $(includedir)/odp/api/plat odpapiplatinclude_HEADERS = \ include/odp/api/plat/atomic_inlines.h \ include/odp/api/plat/buffer_inlines.h \ + include/odp/api/plat/buffer_inline_types.h \ include/odp/api/plat/byteorder_inlines.h \ include/odp/api/plat/cpu_inlines.h \ + include/odp/api/plat/debug_inlines.h \ include/odp/api/plat/event_inlines.h \ include/odp/api/plat/event_inline_types.h \ include/odp/api/plat/event_vector_inline_types.h \ @@ -46,12 +48,17 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/pool_inline_types.h \ include/odp/api/plat/queue_inlines.h \ include/odp/api/plat/queue_inline_types.h \ + include/odp/api/plat/rwlock_inlines.h \ + include/odp/api/plat/rwlock_recursive_inlines.h \ + include/odp/api/plat/schedule_inlines.h \ + include/odp/api/plat/schedule_inline_types.h \ include/odp/api/plat/spinlock_inlines.h \ include/odp/api/plat/spinlock_recursive_inlines.h \ include/odp/api/plat/std_inlines.h \ include/odp/api/plat/strong_types.h \ include/odp/api/plat/sync_inlines.h \ include/odp/api/plat/thread_inlines.h \ + include/odp/api/plat/thread_inline_types.h \ include/odp/api/plat/ticketlock_inlines.h \ include/odp/api/plat/time_inlines.h \ include/odp/api/plat/timer_inlines.h \ @@ -225,8 +232,6 @@ __LIB__libodp_linux_la_SOURCES = \ odp_random.c \ odp_random_std.c \ odp_random_openssl.c \ - odp_rwlock.c \ - odp_rwlock_recursive.c \ odp_schedule_basic.c \ odp_schedule_if.c \ odp_schedule_scalable.c \ @@ -288,6 +293,9 @@ __LIB__libodp_linux_la_SOURCES += \ odp_packet_io_api.c \ odp_pool_api.c \ odp_queue_api.c \ + odp_rwlock_api.c \ + odp_rwlock_recursive_api.c \ + odp_schedule_api.c \ odp_spinlock_api.c \ odp_spinlock_recursive_api.c \ odp_std_api.c \ diff --git a/platform/linux-generic/arch/aarch64/cpu_flags.c b/platform/linux-generic/arch/aarch64/cpu_flags.c index e013f749f..9c2e9fa45 100644 --- a/platform/linux-generic/arch/aarch64/cpu_flags.c +++ b/platform/linux-generic/arch/aarch64/cpu_flags.c @@ -494,363 +494,363 @@ static void _odp_sys_info_print_acle_flags(void) (void)ndef; /* See ARM C Language Extensions documentation for details */ - ODP_PRINT("ARM FEATURES:\n"); + _ODP_PRINT("ARM FEATURES:\n"); - ODP_PRINT(" __ARM_ALIGN_MAX_PWR "); + _ODP_PRINT(" __ARM_ALIGN_MAX_PWR "); #ifdef __ARM_ALIGN_MAX_PWR - ODP_PRINT("%i\n", __ARM_ALIGN_MAX_PWR); + _ODP_PRINT("%i\n", __ARM_ALIGN_MAX_PWR); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_ALIGN_MAX_STACK_PWR "); + _ODP_PRINT(" __ARM_ALIGN_MAX_STACK_PWR "); #ifdef __ARM_ALIGN_MAX_STACK_PWR - ODP_PRINT("%i\n", __ARM_ALIGN_MAX_STACK_PWR); + _ODP_PRINT("%i\n", __ARM_ALIGN_MAX_STACK_PWR); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_ARCH "); + _ODP_PRINT(" __ARM_ARCH "); #ifdef __ARM_ARCH - ODP_PRINT("%i\n", __ARM_ARCH); + _ODP_PRINT("%i\n", __ARM_ARCH); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_ARCH_ISA_A64 "); + _ODP_PRINT(" __ARM_ARCH_ISA_A64 "); #ifdef __ARM_ARCH_ISA_A64 - ODP_PRINT("%i\n", __ARM_ARCH_ISA_A64); + _ODP_PRINT("%i\n", __ARM_ARCH_ISA_A64); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_BIG_ENDIAN "); + _ODP_PRINT(" __ARM_BIG_ENDIAN "); #ifdef __ARM_BIG_ENDIAN - ODP_PRINT("%i\n", __ARM_BIG_ENDIAN); + _ODP_PRINT("%i\n", __ARM_BIG_ENDIAN); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_BF16_FORMAT_ALTERNATIVE "); + _ODP_PRINT(" __ARM_BF16_FORMAT_ALTERNATIVE "); #ifdef __ARM_BF16_FORMAT_ALTERNATIVE - ODP_PRINT("%i\n", __ARM_BF16_FORMAT_ALTERNATIVE); + _ODP_PRINT("%i\n", __ARM_BF16_FORMAT_ALTERNATIVE); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_ATOMICS "); + _ODP_PRINT(" __ARM_FEATURE_ATOMICS "); #ifdef __ARM_FEATURE_ATOMICS - ODP_PRINT("%i\n", __ARM_FEATURE_ATOMICS); + _ODP_PRINT("%i\n", __ARM_FEATURE_ATOMICS); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_BF16 "); + _ODP_PRINT(" __ARM_FEATURE_BF16 "); #ifdef __ARM_FEATURE_BF16 - ODP_PRINT("%i\n", __ARM_FEATURE_BF16); + _ODP_PRINT("%i\n", __ARM_FEATURE_BF16); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_BTI_DEFAULT "); + _ODP_PRINT(" __ARM_FEATURE_BTI_DEFAULT "); #ifdef __ARM_FEATURE_BTI_DEFAULT - ODP_PRINT("%i\n", __ARM_FEATURE_BTI_DEFAULT); + _ODP_PRINT("%i\n", __ARM_FEATURE_BTI_DEFAULT); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_CDE "); + _ODP_PRINT(" __ARM_FEATURE_CDE "); #ifdef __ARM_FEATURE_CDE - ODP_PRINT("%i\n", __ARM_FEATURE_CDE); + _ODP_PRINT("%i\n", __ARM_FEATURE_CDE); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_CDE_COPROC "); + _ODP_PRINT(" __ARM_FEATURE_CDE_COPROC "); #ifdef __ARM_FEATURE_CDE_COPROC - ODP_PRINT("0x%X\n", __ARM_FEATURE_CDE_COPROC); + _ODP_PRINT("0x%X\n", __ARM_FEATURE_CDE_COPROC); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_CLZ "); + _ODP_PRINT(" __ARM_FEATURE_CLZ "); #ifdef __ARM_FEATURE_CLZ - ODP_PRINT("%i\n", __ARM_FEATURE_CLZ); + _ODP_PRINT("%i\n", __ARM_FEATURE_CLZ); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_COMPLEX "); + _ODP_PRINT(" __ARM_FEATURE_COMPLEX "); #ifdef __ARM_FEATURE_COMPLEX - ODP_PRINT("%i\n", __ARM_FEATURE_COMPLEX); + _ODP_PRINT("%i\n", __ARM_FEATURE_COMPLEX); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_COPROC "); + _ODP_PRINT(" __ARM_FEATURE_COPROC "); #ifdef __ARM_FEATURE_COPROC - ODP_PRINT("0x%X\n", __ARM_FEATURE_COPROC); + _ODP_PRINT("0x%X\n", __ARM_FEATURE_COPROC); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_CRC32 "); + _ODP_PRINT(" __ARM_FEATURE_CRC32 "); #ifdef __ARM_FEATURE_CRC32 - ODP_PRINT("%i\n", __ARM_FEATURE_CRC32); + _ODP_PRINT("%i\n", __ARM_FEATURE_CRC32); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_CRYPTO "); + _ODP_PRINT(" __ARM_FEATURE_CRYPTO "); #ifdef __ARM_FEATURE_CRYPTO - ODP_PRINT("%i\n", __ARM_FEATURE_CRYPTO); + _ODP_PRINT("%i\n", __ARM_FEATURE_CRYPTO); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_DIRECTED_ROUNDING "); + _ODP_PRINT(" __ARM_FEATURE_DIRECTED_ROUNDING "); #ifdef __ARM_FEATURE_DIRECTED_ROUNDING - ODP_PRINT("%i\n", __ARM_FEATURE_DIRECTED_ROUNDING); + _ODP_PRINT("%i\n", __ARM_FEATURE_DIRECTED_ROUNDING); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_DOTPROD "); + _ODP_PRINT(" __ARM_FEATURE_DOTPROD "); #ifdef __ARM_FEATURE_DOTPROD - ODP_PRINT("%i\n", __ARM_FEATURE_DOTPROD); + _ODP_PRINT("%i\n", __ARM_FEATURE_DOTPROD); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_FMA "); + _ODP_PRINT(" __ARM_FEATURE_FMA "); #ifdef __ARM_FEATURE_FMA - ODP_PRINT("%i\n", __ARM_FEATURE_FMA); + _ODP_PRINT("%i\n", __ARM_FEATURE_FMA); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_FP16_FML "); + _ODP_PRINT(" __ARM_FEATURE_FP16_FML "); #ifdef __ARM_FEATURE_FP16_FML - ODP_PRINT("%i\n", __ARM_FEATURE_FP16_FML); + _ODP_PRINT("%i\n", __ARM_FEATURE_FP16_FML); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_FRINT "); + _ODP_PRINT(" __ARM_FEATURE_FRINT "); #ifdef __ARM_FEATURE_FRINT - ODP_PRINT("%i\n", __ARM_FEATURE_FRINT); + _ODP_PRINT("%i\n", __ARM_FEATURE_FRINT); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_IDIV "); + _ODP_PRINT(" __ARM_FEATURE_IDIV "); #ifdef __ARM_FEATURE_IDIV - ODP_PRINT("%i\n", __ARM_FEATURE_IDIV); + _ODP_PRINT("%i\n", __ARM_FEATURE_IDIV); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_JCVT "); + _ODP_PRINT(" __ARM_FEATURE_JCVT "); #ifdef __ARM_FEATURE_JCVT - ODP_PRINT("%i\n", __ARM_FEATURE_JCVT); + _ODP_PRINT("%i\n", __ARM_FEATURE_JCVT); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_MATMUL_INT8 "); + _ODP_PRINT(" __ARM_FEATURE_MATMUL_INT8 "); #ifdef __ARM_FEATURE_MATMUL_INT8 - ODP_PRINT("%i\n", __ARM_FEATURE_MATMUL_INT8); + _ODP_PRINT("%i\n", __ARM_FEATURE_MATMUL_INT8); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_MEMORY_TAGGING "); + _ODP_PRINT(" __ARM_FEATURE_MEMORY_TAGGING "); #ifdef __ARM_FEATURE_MEMORY_TAGGING - ODP_PRINT("%i\n", __ARM_FEATURE_MEMORY_TAGGING); + _ODP_PRINT("%i\n", __ARM_FEATURE_MEMORY_TAGGING); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_MVE "); + _ODP_PRINT(" __ARM_FEATURE_MVE "); #ifdef __ARM_FEATURE_MVE - ODP_PRINT("0x%X\n", __ARM_FEATURE_MVE); + _ODP_PRINT("0x%X\n", __ARM_FEATURE_MVE); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_NUMERIC_MAXMIN "); + _ODP_PRINT(" __ARM_FEATURE_NUMERIC_MAXMIN "); #ifdef __ARM_FEATURE_NUMERIC_MAXMIN - ODP_PRINT("%i\n", __ARM_FEATURE_NUMERIC_MAXMIN); + _ODP_PRINT("%i\n", __ARM_FEATURE_NUMERIC_MAXMIN); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_PAC_DEFAULT "); + _ODP_PRINT(" __ARM_FEATURE_PAC_DEFAULT "); #ifdef __ARM_FEATURE_PAC_DEFAULT - ODP_PRINT("0x%X\n", __ARM_FEATURE_PAC_DEFAULT); + _ODP_PRINT("0x%X\n", __ARM_FEATURE_PAC_DEFAULT); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_QRDMX "); + _ODP_PRINT(" __ARM_FEATURE_QRDMX "); #ifdef __ARM_FEATURE_QRDMX - ODP_PRINT("%i\n", __ARM_FEATURE_QRDMX); + _ODP_PRINT("%i\n", __ARM_FEATURE_QRDMX); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_RNG "); + _ODP_PRINT(" __ARM_FEATURE_RNG "); #ifdef __ARM_FEATURE_RNG - ODP_PRINT("%i\n", __ARM_FEATURE_RNG); + _ODP_PRINT("%i\n", __ARM_FEATURE_RNG); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_SHA3 "); + _ODP_PRINT(" __ARM_FEATURE_SHA3 "); #ifdef __ARM_FEATURE_SHA3 - ODP_PRINT("%i\n", __ARM_FEATURE_SHA3); + _ODP_PRINT("%i\n", __ARM_FEATURE_SHA3); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_SHA512 "); + _ODP_PRINT(" __ARM_FEATURE_SHA512 "); #ifdef __ARM_FEATURE_SHA512 - ODP_PRINT("%i\n", __ARM_FEATURE_SHA512); + _ODP_PRINT("%i\n", __ARM_FEATURE_SHA512); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_SM3 "); + _ODP_PRINT(" __ARM_FEATURE_SM3 "); #ifdef __ARM_FEATURE_SM3 - ODP_PRINT("%i\n", __ARM_FEATURE_SM3); + _ODP_PRINT("%i\n", __ARM_FEATURE_SM3); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_SM4 "); + _ODP_PRINT(" __ARM_FEATURE_SM4 "); #ifdef __ARM_FEATURE_SM4 - ODP_PRINT("%i\n", __ARM_FEATURE_SM4); + _ODP_PRINT("%i\n", __ARM_FEATURE_SM4); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_TME "); + _ODP_PRINT(" __ARM_FEATURE_TME "); #ifdef __ARM_FEATURE_TME - ODP_PRINT("%i\n", __ARM_FEATURE_TME); + _ODP_PRINT("%i\n", __ARM_FEATURE_TME); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FEATURE_UNALIGNED "); + _ODP_PRINT(" __ARM_FEATURE_UNALIGNED "); #ifdef __ARM_FEATURE_UNALIGNED - ODP_PRINT("%i\n", __ARM_FEATURE_UNALIGNED); + _ODP_PRINT("%i\n", __ARM_FEATURE_UNALIGNED); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FP "); + _ODP_PRINT(" __ARM_FP "); #ifdef __ARM_FP - ODP_PRINT("0x%X\n", __ARM_FP); + _ODP_PRINT("0x%X\n", __ARM_FP); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FP_FAST "); + _ODP_PRINT(" __ARM_FP_FAST "); #ifdef __ARM_FP_FAST - ODP_PRINT("%i\n", __ARM_FP_FAST); + _ODP_PRINT("%i\n", __ARM_FP_FAST); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FP_FENV_ROUNDING "); + _ODP_PRINT(" __ARM_FP_FENV_ROUNDING "); #ifdef __ARM_FP_FENV_ROUNDING - ODP_PRINT("%i\n", __ARM_FP_FENV_ROUNDING); + _ODP_PRINT("%i\n", __ARM_FP_FENV_ROUNDING); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FP16_ARGS "); + _ODP_PRINT(" __ARM_FP16_ARGS "); #ifdef __ARM_FP16_ARGS - ODP_PRINT("%i\n", __ARM_FP16_ARGS); + _ODP_PRINT("%i\n", __ARM_FP16_ARGS); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FP16_FORMAT_ALTERNATIVE "); + _ODP_PRINT(" __ARM_FP16_FORMAT_ALTERNATIVE "); #ifdef __ARM_FP16_FORMAT_ALTERNATIVE - ODP_PRINT("%i\n", __ARM_FP16_FORMAT_ALTERNATIVE); + _ODP_PRINT("%i\n", __ARM_FP16_FORMAT_ALTERNATIVE); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_FP16_FORMAT_IEEE "); + _ODP_PRINT(" __ARM_FP16_FORMAT_IEEE "); #ifdef __ARM_FP16_FORMAT_IEEE - ODP_PRINT("%i\n", __ARM_FP16_FORMAT_IEEE); + _ODP_PRINT("%i\n", __ARM_FP16_FORMAT_IEEE); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_NEON "); + _ODP_PRINT(" __ARM_NEON "); #ifdef __ARM_NEON - ODP_PRINT("%i\n", __ARM_NEON); + _ODP_PRINT("%i\n", __ARM_NEON); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_NEON_FP "); + _ODP_PRINT(" __ARM_NEON_FP "); #ifdef __ARM_NEON_FP - ODP_PRINT("0x%X\n", __ARM_NEON_FP); + _ODP_PRINT("0x%X\n", __ARM_NEON_FP); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_PCS_AAPCS64 "); + _ODP_PRINT(" __ARM_PCS_AAPCS64 "); #ifdef __ARM_PCS_AAPCS64 - ODP_PRINT("%i\n", __ARM_PCS_AAPCS64); + _ODP_PRINT("%i\n", __ARM_PCS_AAPCS64); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_ROPI "); + _ODP_PRINT(" __ARM_ROPI "); #ifdef __ARM_ROPI - ODP_PRINT("%i\n", __ARM_ROPI); + _ODP_PRINT("%i\n", __ARM_ROPI); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_RWPI "); + _ODP_PRINT(" __ARM_RWPI "); #ifdef __ARM_RWPI - ODP_PRINT("%i\n", __ARM_RWPI); + _ODP_PRINT("%i\n", __ARM_RWPI); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_SIZEOF_MINIMAL_ENUM "); + _ODP_PRINT(" __ARM_SIZEOF_MINIMAL_ENUM "); #ifdef __ARM_SIZEOF_MINIMAL_ENUM - ODP_PRINT("%i\n", __ARM_SIZEOF_MINIMAL_ENUM); + _ODP_PRINT("%i\n", __ARM_SIZEOF_MINIMAL_ENUM); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" __ARM_SIZEOF_WCHAR_T "); + _ODP_PRINT(" __ARM_SIZEOF_WCHAR_T "); #ifdef __ARM_SIZEOF_WCHAR_T - ODP_PRINT("%i\n", __ARM_SIZEOF_WCHAR_T); + _ODP_PRINT("%i\n", __ARM_SIZEOF_WCHAR_T); #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT(" ARM ISA version: "); + _ODP_PRINT(" ARM ISA version: "); #if defined(__ARM_ARCH) if (__ARM_ARCH < 8) { - ODP_PRINT("v%i\n", __ARM_ARCH); + _ODP_PRINT("v%i\n", __ARM_ARCH); } else if (__ARM_ARCH == 8) { /* Actually, this checks for new NEON instructions in * v8.1, but is currently the only way to distinguish * v8.0 and >=v8.1. */ #ifdef __ARM_FEATURE_QRDMX - ODP_PRINT("v8.1 or higher\n"); + _ODP_PRINT("v8.1 or higher\n"); #else - ODP_PRINT("v8.0\n"); + _ODP_PRINT("v8.0\n"); #endif } else { /* ACLE 2018 defines that from v8.1 onwards the value includes @@ -859,13 +859,13 @@ static void _odp_sys_info_print_acle_flags(void) int major = __ARM_ARCH / 100; int minor = __ARM_ARCH - (major * 100); - ODP_PRINT("v%i.%i\n", major, minor); + _ODP_PRINT("v%i.%i\n", major, minor); } #else - ODP_PRINT("%s\n", ndef); + _ODP_PRINT("%s\n", ndef); #endif - ODP_PRINT("\n"); + _ODP_PRINT("\n"); } static int check_hwcap_duplicates(unsigned int hwcap_field) @@ -893,7 +893,7 @@ static void _odp_sys_info_print_hwcap_flags(void) unsigned long hwcaps, hwcaps2; unsigned int size, size2; - ODP_PRINT("ARM FEATURES SUPPORTED BY HARDWARE:\n"); + _ODP_PRINT("ARM FEATURES SUPPORTED BY HARDWARE:\n"); /* Print supported hardware flags via AT_HWCAP entry of the hwcaps * auxiliary vector. */ @@ -907,7 +907,7 @@ static void _odp_sys_info_print_hwcap_flags(void) } if (hwcaps & 0x01) - ODP_PRINT("%s ", hwcap_flags[i].feat_flag); + _ODP_PRINT("%s ", hwcap_flags[i].feat_flag); hwcaps = hwcaps >> 1; } } @@ -919,18 +919,18 @@ static void _odp_sys_info_print_hwcap_flags(void) for (unsigned long i = 0; i < size2; i++) { if (hwcap2_flags[i].valid) { if (hwcaps2 & 0x01) - ODP_PRINT("%s ", hwcap2_flags[i].feat_flag); + _ODP_PRINT("%s ", hwcap2_flags[i].feat_flag); hwcaps2 = hwcaps2 >> 1; } } - ODP_PRINT("\n"); + _ODP_PRINT("\n"); /* Re-initialize hwcaps and hwcaps2 */ hwcaps = 0; hwcaps2 = 0; - ODP_PRINT("\nARM FEATURES NOT SUPPORTED BY HARDWARE:\n"); + _ODP_PRINT("\nARM FEATURES NOT SUPPORTED BY HARDWARE:\n"); hwcaps = getauxval(AT_HWCAP); for (unsigned long i = 0; i < size; i++) { @@ -941,7 +941,7 @@ static void _odp_sys_info_print_hwcap_flags(void) } if (!(hwcaps & 0x01)) - ODP_PRINT("%s ", hwcap_flags[i].feat_flag); + _ODP_PRINT("%s ", hwcap_flags[i].feat_flag); hwcaps = hwcaps >> 1; } } @@ -950,26 +950,26 @@ static void _odp_sys_info_print_hwcap_flags(void) for (unsigned long i = 0; i < size2; i++) { if (hwcap2_flags[i].valid) { if (!(hwcaps2 & 0x01)) - ODP_PRINT("%s ", hwcap2_flags[i].feat_flag); + _ODP_PRINT("%s ", hwcap2_flags[i].feat_flag); hwcaps2 = hwcaps2 >> 1; } } - ODP_PRINT("\n"); + _ODP_PRINT("\n"); - ODP_PRINT("\nARM FEATURES NOT SUPPORTED BY KERNEL:\n"); + _ODP_PRINT("\nARM FEATURES NOT SUPPORTED BY KERNEL:\n"); for (unsigned long i = 0; i < size; i++) { if (!hwcap_flags[i].valid) - ODP_PRINT("%s ", hwcap_flags[i].feat_flag); + _ODP_PRINT("%s ", hwcap_flags[i].feat_flag); } for (unsigned long i = 0; i < size2; i++) { if (!hwcap2_flags[i].valid) - ODP_PRINT("%s ", hwcap2_flags[i].feat_flag); + _ODP_PRINT("%s ", hwcap2_flags[i].feat_flag); } - ODP_PRINT("\n\n"); + _ODP_PRINT("\n\n"); } void _odp_cpu_flags_print_all(void) diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c index 32341dd02..11fadc971 100644 --- a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c +++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c @@ -7,26 +7,28 @@ */ #include <odp_posix_extensions.h> + #include <odp/api/crypto.h> -#include <odp_init_internal.h> #include <odp/api/spinlock.h> #include <odp/api/sync.h> #include <odp/api/debug.h> #include <odp/api/align.h> #include <odp/api/shared_memory.h> -#include <odp_debug_internal.h> #include <odp/api/hints.h> #include <odp/api/random.h> + +#include <odp/api/plat/event_inlines.h> #include <odp/api/plat/packet_inlines.h> +#include <odp/api/plat/queue_inlines.h> #include <odp/api/plat/thread_inlines.h> + +#include <odp_debug_internal.h> +#include <odp_global_data.h> +#include <odp_init_internal.h> #include <odp_packet_internal.h> -#include <odp/api/plat/queue_inlines.h> #include "AArch64cryptolib.h" -/* Inlined API functions */ -#include <odp/api/plat/event_inlines.h> - #define MAX_SESSIONS 4000 /* Length in bytes */ #define ARM_CRYPTO_MAX_CIPHER_KEY_LENGTH 32 @@ -214,7 +216,7 @@ null_crypto_routine(odp_packet_t pkt ODP_UNUSED, static inline void copy_aad(uint8_t *dst, uint8_t *src, uint32_t len) { - ODP_ASSERT(len == 8 || len == 12); + _ODP_ASSERT(len == 8 || len == 12); /* Use constant length memcpy for better optimization result */ if (len == 8) @@ -246,7 +248,7 @@ void aes_gcm_encrypt(odp_packet_t pkt, /* Fail early if cipher_range is too large */ if (odp_unlikely(in_len > ARM_CRYPTO_MAX_DATA_LENGTH)) { - ODP_DBG("ARM Crypto: Packet size too large for requested operation\n"); + _ODP_DBG("ARM Crypto: Packet size too large for requested operation\n"); goto err; } @@ -259,14 +261,14 @@ void aes_gcm_encrypt(odp_packet_t pkt, goto err; #else iv_ptr = param->cipher_iv_ptr; - ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); + _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); #endif cs.constants = &session->cc; rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs); if (odp_unlikely(rc)) { - ODP_DBG("ARM Crypto: Failure while setting nonce\n"); + _ODP_DBG("ARM Crypto: Failure while setting nonce\n"); goto err; } @@ -300,7 +302,7 @@ void aes_gcm_encrypt(odp_packet_t pkt, data, tag); if (odp_unlikely(rc)) { - ODP_DBG("ARM Crypto: AES GCM Encoding failed\n"); + _ODP_DBG("ARM Crypto: AES GCM Encoding failed\n"); goto err; } @@ -346,7 +348,7 @@ void aes_gcm_decrypt(odp_packet_t pkt, /* Fail early if cipher_range is too large */ if (odp_unlikely(in_len > ARM_CRYPTO_MAX_DATA_LENGTH)) { - ODP_DBG("ARM Crypto: Packet size too large for requested operation\n"); + _ODP_DBG("ARM Crypto: Packet size too large for requested operation\n"); goto err; } @@ -359,14 +361,14 @@ void aes_gcm_decrypt(odp_packet_t pkt, goto err; #else iv_ptr = param->cipher_iv_ptr; - ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); + _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); #endif cs.constants = &session->cc; rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs); if (odp_unlikely(rc)) { - ODP_DBG("ARM Crypto: Failure while setting nonce\n"); + _ODP_DBG("ARM Crypto: Failure while setting nonce\n"); goto err; } @@ -404,7 +406,7 @@ void aes_gcm_decrypt(odp_packet_t pkt, tag, data); if (odp_unlikely(rc)) { - ODP_DBG("ARM Crypto: AES GCM Decoding failed\n"); + _ODP_DBG("ARM Crypto: AES GCM Decoding failed\n"); goto err; } @@ -545,7 +547,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, odp_crypto_generic_session_t *session; if (odp_global_ro.disable.crypto) { - ODP_ERR("Crypto is disabled\n"); + _ODP_ERR("Crypto is disabled\n"); /* Dummy output to avoid compiler warning about uninitialized * variables */ *status = ODP_CRYPTO_SES_ERR_ENOMEM; @@ -564,13 +566,13 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, session->p = *param; if (session->p.cipher_iv_len > ARM_CRYPTO_MAX_IV_LENGTH) { - ODP_DBG("Maximum IV length exceeded\n"); + _ODP_DBG("Maximum IV length exceeded\n"); *status = ODP_CRYPTO_SES_ERR_CIPHER; goto err; } if (session->p.auth_iv_len > ARM_CRYPTO_MAX_IV_LENGTH) { - ODP_DBG("Maximum auth IV length exceeded\n"); + _ODP_DBG("Maximum auth IV length exceeded\n"); *status = ODP_CRYPTO_SES_ERR_CIPHER; goto err; } @@ -622,7 +624,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, session->p.auth_digest_len, session->p.cipher_key.data, &session->cc) != 0) { - ODP_DBG("ARM Crypto: Failure in setting constants\n"); + _ODP_DBG("ARM Crypto: Failure in setting constants\n"); rc = -1; } rc = process_aes_gcm_param(session); @@ -754,7 +756,7 @@ int _odp_crypto_init_global(void) int idx; if (odp_global_ro.disable.crypto) { - ODP_PRINT("\nODP crypto is DISABLED\n"); + _ODP_PRINT("\nODP crypto is DISABLED\n"); return 0; } @@ -766,7 +768,7 @@ int _odp_crypto_init_global(void) ODP_CACHE_LINE_SIZE, 0); if (ODP_SHM_INVALID == shm) { - ODP_ERR("unable to allocate crypto pool\n"); + _ODP_ERR("unable to allocate crypto pool\n"); return -1; } @@ -798,13 +800,13 @@ int _odp_crypto_term_global(void) for (session = global->free; session != NULL; session = session->next) count++; if (count != MAX_SESSIONS) { - ODP_ERR("crypto sessions still active\n"); + _ODP_ERR("crypto sessions still active\n"); rc = -1; } ret = odp_shm_free(odp_shm_lookup("_odp_crypto_pool_armv8crypto")); if (ret < 0) { - ODP_ERR("shm free failed for _odp_crypto_pool_armv8crypto\n"); + _ODP_ERR("shm free failed for _odp_crypto_pool_armv8crypto\n"); rc = -1; } @@ -831,7 +833,7 @@ odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev) { /* This check not mandated by the API specification */ if (odp_event_type(ev) != ODP_EVENT_CRYPTO_COMPL) - ODP_ABORT("Event not a crypto completion"); + _ODP_ABORT("Event not a crypto completion"); return (odp_crypto_compl_t)ev; } @@ -848,7 +850,7 @@ odp_crypto_compl_result(odp_crypto_compl_t completion_event, (void)result; /* We won't get such events anyway, so there can be no result */ - ODP_ASSERT(0); + _ODP_ASSERT(0); } void @@ -878,8 +880,8 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) odp_packet_t odp_crypto_packet_from_event(odp_event_t ev) { /* This check not mandated by the API specification */ - ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); - ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); + _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); return odp_packet_from_event(ev); } @@ -894,7 +896,7 @@ int odp_crypto_result(odp_crypto_packet_result_t *result, { odp_crypto_packet_result_t *op_result; - ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == + _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == ODP_EVENT_PACKET_CRYPTO); op_result = get_op_result_from_packet(packet); @@ -912,13 +914,13 @@ static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src) md_copy = _odp_packet_copy_md_possible(odp_packet_pool(dst), odp_packet_pool(src)); if (odp_unlikely(md_copy < 0)) { - ODP_ERR("Unable to copy packet metadata\n"); + _ODP_ERR("Unable to copy packet metadata\n"); return -1; } rc = odp_packet_copy_from_pkt(dst, 0, src, 0, odp_packet_len(src)); if (odp_unlikely(rc < 0)) { - ODP_ERR("Unable to copy packet data\n"); + _ODP_ERR("Unable to copy packet data\n"); return -1; } @@ -938,7 +940,7 @@ static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *sessio if (pkt_out == ODP_PACKET_INVALID) { odp_pool_t pool = session->p.output_pool; - ODP_ASSERT(pool != ODP_POOL_INVALID); + _ODP_ASSERT(pool != ODP_POOL_INVALID); if (pool == odp_packet_pool(pkt_in)) { pkt_out = pkt_in; } else { @@ -991,7 +993,7 @@ int odp_crypto_op(const odp_packet_t pkt_in[], for (i = 0; i < num_pkt; i++) { session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session; - ODP_ASSERT(ODP_CRYPTO_SYNC == session->p.op_mode); + _ODP_ASSERT(ODP_CRYPTO_SYNC == session->p.op_mode); rc = crypto_int(pkt_in[i], &pkt_out[i], ¶m[i]); if (rc < 0) @@ -1013,8 +1015,8 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[], for (i = 0; i < num_pkt; i++) { session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session; - ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); - ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); + _ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); + _ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); pkt = pkt_out[i]; rc = crypto_int(pkt_in[i], &pkt, ¶m[i]); diff --git a/platform/linux-generic/arch/aarch64/odp_llsc.h b/platform/linux-generic/arch/aarch64/odp_llsc.h index 1c2610511..ba7a447d4 100644 --- a/platform/linux-generic/arch/aarch64/odp_llsc.h +++ b/platform/linux-generic/arch/aarch64/odp_llsc.h @@ -30,7 +30,7 @@ static inline uint16_t ll8(uint8_t *var, int mm) : "r" (var) : ); else - ODP_ABORT(); + _ODP_ABORT(); return old; } @@ -49,7 +49,7 @@ static inline uint32_t ll32(uint32_t *var, int mm) : "r" (var) : ); else - ODP_ABORT(); + _ODP_ABORT(); return old; } @@ -69,7 +69,7 @@ static inline uint32_t sc32(uint32_t *var, uint32_t neu, int mm) : "r" (neu), "r" (var) : ); else - ODP_ABORT(); + _ODP_ABORT(); return ret; } @@ -88,7 +88,7 @@ static inline uint64_t ll(uint64_t *var, int mm) : "r" (var) : ); else - ODP_ABORT(); + _ODP_ABORT(); return old; } @@ -110,7 +110,7 @@ static inline uint32_t sc(uint64_t *var, uint64_t neu, int mm) : "r" (neu), "r" (var) : ); else - ODP_ABORT(); + _ODP_ABORT(); return ret; } @@ -136,7 +136,7 @@ static inline _odp_u128_t lld(_odp_u128_t *var, int mm) : "r" (var) : ); else - ODP_ABORT(); + _ODP_ABORT(); return old.i128; } @@ -162,7 +162,7 @@ static inline uint32_t scd(_odp_u128_t *var, _odp_u128_t neu, int mm) "r" (var) : ); else - ODP_ABORT(); + _ODP_ABORT(); #pragma GCC diagnostic pop return ret; } diff --git a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c index 921bafaf0..25e3e4fc6 100644 --- a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c @@ -336,7 +336,7 @@ int _odp_cpuinfo_parser(FILE *file, system_info_t *sysinfo) if (sysinfo->cpu_hz_max[id] == 0) { uint64_t hz = sysinfo->default_cpu_hz_max; - ODP_PRINT("WARN: cpu[%i] uses default max " + _ODP_PRINT("WARN: cpu[%i] uses default max " "frequency of %" PRIu64 " Hz from " "config file\n", id, hz); sysinfo->cpu_hz_max[id] = hz; diff --git a/platform/linux-generic/arch/default/odp_cpu_cycles.c b/platform/linux-generic/arch/default/odp_cpu_cycles.c index 5d0d5db1d..41436a672 100644 --- a/platform/linux-generic/arch/default/odp_cpu_cycles.c +++ b/platform/linux-generic/arch/default/odp_cpu_cycles.c @@ -28,7 +28,7 @@ uint64_t _odp_cpu_cycles(void) ret = clock_gettime(CLOCK_MONOTONIC_RAW, &time); if (ret != 0) - ODP_ABORT("clock_gettime failed\n"); + _ODP_ABORT("clock_gettime failed\n"); hz = odp_global_ro.system_info.cpu_hz_max[0]; diff --git a/platform/linux-generic/arch/x86/cpu_flags.c b/platform/linux-generic/arch/x86/cpu_flags.c index c75a87233..036645dbc 100644 --- a/platform/linux-generic/arch/x86/cpu_flags.c +++ b/platform/linux-generic/arch/x86/cpu_flags.c @@ -356,7 +356,7 @@ void _odp_cpu_flags_print_all(void) len += snprintf(&str[len], max_len - len, "\n\n"); str[len] = '\0'; - ODP_PRINT("%s", str); + _ODP_PRINT("%s", str); } int _odp_cpu_has_global_time(void) diff --git a/platform/linux-generic/arch/x86/odp_cpu_cycles.c b/platform/linux-generic/arch/x86/odp_cpu_cycles.c index 406668a79..2624af0f6 100644 --- a/platform/linux-generic/arch/x86/odp_cpu_cycles.c +++ b/platform/linux-generic/arch/x86/odp_cpu_cycles.c @@ -13,7 +13,7 @@ int _odp_cpu_cycles_init_global(void) { if (_odp_cpu_flags_has_rdtsc() == 0) { - ODP_ERR("RDTSC instruction not supported\n"); + _ODP_ERR("RDTSC instruction not supported\n"); return -1; } diff --git a/platform/linux-generic/arch/x86/odp_global_time.c b/platform/linux-generic/arch/x86/odp_global_time.c index d4d7ab150..00e3a1ac1 100644 --- a/platform/linux-generic/arch/x86/odp_global_time.c +++ b/platform/linux-generic/arch/x86/odp_global_time.c @@ -34,19 +34,19 @@ uint64_t _odp_cpu_global_time_freq(void) sleep.tv_nsec = SEC_IN_NS / 4; if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) { - ODP_DBG("clock_gettime failed\n"); + _ODP_DBG("clock_gettime failed\n"); return 0; } t1 = _odp_cpu_global_time(); if (nanosleep(&sleep, NULL) < 0) { - ODP_DBG("nanosleep failed\n"); + _ODP_DBG("nanosleep failed\n"); return 0; } if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) { - ODP_DBG("clock_gettime failed\n"); + _ODP_DBG("clock_gettime failed\n"); return 0; } diff --git a/platform/linux-generic/include-abi/odp/api/abi/rwlock.h b/platform/linux-generic/include-abi/odp/api/abi/rwlock.h index 96dbc9b83..78d7c1a4a 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/rwlock.h +++ b/platform/linux-generic/include-abi/odp/api/abi/rwlock.h @@ -5,3 +5,6 @@ */ #include <odp/api/abi-default/rwlock.h> + +/* Inlined API functions */ +#include <odp/api/plat/rwlock_inlines.h> diff --git a/platform/linux-generic/include-abi/odp/api/abi/rwlock_recursive.h b/platform/linux-generic/include-abi/odp/api/abi/rwlock_recursive.h index 14c964c9d..ab7150605 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/rwlock_recursive.h +++ b/platform/linux-generic/include-abi/odp/api/abi/rwlock_recursive.h @@ -5,3 +5,6 @@ */ #include <odp/api/abi-default/rwlock_recursive.h> + +/* Inlined API functions */ +#include <odp/api/plat/rwlock_recursive_inlines.h> diff --git a/platform/linux-generic/include-abi/odp/api/abi/schedule.h b/platform/linux-generic/include-abi/odp/api/abi/schedule.h index 748c6aa8a..bb28886b0 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/schedule.h +++ b/platform/linux-generic/include-abi/odp/api/abi/schedule.h @@ -1,7 +1,28 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#include <odp/api/abi-default/schedule.h> +/** + * @file + * + * ODP Schedule + */ + +#ifndef ODP_API_ABI_SCHEDULE_H_ +#define ODP_API_ABI_SCHEDULE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Inlined API functions */ +#include <odp/api/plat/schedule_inlines.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h b/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h new file mode 100644 index 000000000..9689ddd06 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_BUFFER_INLINE_TYPES_H_ +#define ODP_PLAT_BUFFER_INLINE_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +/* Buffer header field accessors */ +#define _odp_buffer_get(buffer_hdr, cast, field) \ + (*(cast *)(uintptr_t)((uint8_t *)buffer_hdr + \ + _odp_buffer_inline_offset.field)) + +/* Buffer header field offsets for inline functions */ +typedef struct _odp_buffer_inline_offset_t { + uint16_t uarea_addr; + +} _odp_buffer_inline_offset_t; + +/** @endcond */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h index 576eb846e..d876f60f4 100644 --- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h @@ -7,16 +7,21 @@ #ifndef ODP_PLAT_BUFFER_INLINES_H_ #define ODP_PLAT_BUFFER_INLINES_H_ -#include <odp/api/event_types.h> +#include <odp/api/event.h> #include <odp/api/pool_types.h> #include <odp/api/abi/buffer.h> +#include <odp/api/plat/buffer_inline_types.h> +#include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/event_inline_types.h> +#include <odp/api/plat/pool_inline_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ extern const _odp_event_inline_offset_t _odp_event_inline_offset; +extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; +extern const _odp_pool_inline_offset_t _odp_pool_inline; #ifndef _ODP_NO_INLINE /* Inline functions by default */ @@ -24,13 +29,17 @@ extern const _odp_event_inline_offset_t _odp_event_inline_offset; #define odp_buffer_from_event __odp_buffer_from_event #define odp_buffer_to_event __odp_buffer_to_event #define odp_buffer_addr __odp_buffer_addr + #define odp_buffer_size __odp_buffer_size #define odp_buffer_pool __odp_buffer_pool + #define odp_buffer_user_area __odp_buffer_user_area #else #define _ODP_INLINE #endif _ODP_INLINE odp_buffer_t odp_buffer_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_BUFFER); + return (odp_buffer_t)ev; } @@ -44,11 +53,23 @@ _ODP_INLINE void *odp_buffer_addr(odp_buffer_t buf) return _odp_event_hdr_field((odp_event_t)buf, void *, base_data); } +_ODP_INLINE uint32_t odp_buffer_size(odp_buffer_t buf) +{ + odp_pool_t pool = _odp_event_hdr_field(buf, odp_pool_t, pool); + + return _odp_pool_get(pool, uint32_t, seg_len); +} + _ODP_INLINE odp_pool_t odp_buffer_pool(odp_buffer_t buf) { return _odp_event_hdr_field(buf, odp_pool_t, pool); } +_ODP_INLINE void *odp_buffer_user_area(odp_buffer_t buf) +{ + return _odp_buffer_get(buf, void *, uarea_addr); +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp/api/plat/debug_inlines.h b/platform/linux-generic/include/odp/api/plat/debug_inlines.h new file mode 100644 index 000000000..bada48f9b --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/debug_inlines.h @@ -0,0 +1,102 @@ +/* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2020-2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP Debug inlines + * + * @warning These definitions are not part of ODP API, they are for + * implementation internal use only. + */ + +#ifndef ODP_DEBUG_INLINES_H_ +#define ODP_DEBUG_INLINES_H_ + +#include <odp/autoheader_external.h> + +#include <odp/api/hints.h> +#include <odp/api/init.h> + +#include <odp/api/plat/thread_inline_types.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Avoid "ISO C99 requires at least one argument for the "..." in a variadic + * macro" errors when building with 'pedantic' option. */ +#pragma GCC system_header + +extern odp_log_func_t ODP_PRINTF_FORMAT(2, 3) _odp_log_fn; +extern odp_abort_func_t _odp_abort_fn; + +#define _ODP_LOG_FN(level, fmt, ...) \ + do { \ + if (_odp_this_thread && _odp_this_thread->log_fn) \ + _odp_this_thread->log_fn(level, fmt, ##__VA_ARGS__); \ + else \ + _odp_log_fn(level, fmt, ##__VA_ARGS__); \ + } while (0) + +/** + * ODP LOG macro. + */ +#define _ODP_LOG(level, fmt, ...) \ + _ODP_LOG_FN(level, "%s:%d:%s():" fmt, __FILE__, \ + __LINE__, __func__, ##__VA_ARGS__) + +/** + * Runtime assertion-macro - aborts if 'cond' is false. + */ +#define _ODP_ASSERT(cond) \ + do { if ((ODP_DEBUG == 1) && (!(cond))) { \ + _ODP_ERR("%s\n", #cond); \ + _odp_abort_fn(); } \ + } while (0) + +/* + * Print debug message to log, if ODP_DEBUG_PRINT flag is set (ignores CONFIG_DEBUG_LEVEL). + */ +#define _ODP_DBG(fmt, ...) \ + do { \ + if (ODP_DEBUG_PRINT == 1) \ + _ODP_LOG(ODP_LOG_DBG, fmt, ##__VA_ARGS__);\ + } while (0) + +/** + * Log error message. + */ +#define _ODP_ERR(fmt, ...) \ + _ODP_LOG(ODP_LOG_ERR, fmt, ##__VA_ARGS__) + +/** + * Log abort message and then stop execution (by default call abort()). + * This function should not return. + */ +#define _ODP_ABORT(fmt, ...) \ + do { \ + _ODP_LOG(ODP_LOG_ABORT, fmt, ##__VA_ARGS__); \ + _odp_abort_fn(); \ + } while (0) + +/** + * Log print message when the application calls one of the ODP APIs + * specifically for dumping internal data. + */ +#define _ODP_PRINT(fmt, ...) \ + _ODP_LOG_FN(ODP_LOG_PRINT, fmt, ##__VA_ARGS__) + +#ifdef __cplusplus +} +#endif + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index 27c2a82fd..dd8e9249c 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -9,13 +9,15 @@ #define ODP_PLAT_EVENT_INLINES_H_ #include <odp/api/event_types.h> -#include <odp/api/packet.h> +#include <odp/api/packet_types.h> #include <odp/api/plat/event_inline_types.h> +#include <odp/api/plat/packet_inline_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ extern const _odp_event_inline_offset_t _odp_event_inline_offset; +extern const _odp_packet_inline_offset_t _odp_packet_inline; #ifndef _ODP_NO_INLINE /* Inline functions by default */ @@ -26,8 +28,6 @@ extern const _odp_event_inline_offset_t _odp_event_inline_offset; #define odp_event_types __odp_event_types #define odp_event_flow_id __odp_event_flow_id #define odp_event_flow_id_set __odp_event_flow_id_set - - #include <odp/api/plat/packet_inlines.h> #else #define _ODP_INLINE #endif @@ -67,7 +67,7 @@ _ODP_INLINE odp_event_subtype_t odp_event_subtype(odp_event_t event) if (__odp_event_type_get(event) != ODP_EVENT_PACKET) return ODP_EVENT_NO_SUBTYPE; - return odp_packet_subtype(odp_packet_from_event(event)); + return (odp_event_subtype_t)_odp_pkt_get((odp_packet_t)event, int8_t, subtype); } _ODP_INLINE odp_event_type_t odp_event_types(odp_event_t event, @@ -76,7 +76,7 @@ _ODP_INLINE odp_event_type_t odp_event_types(odp_event_t event, odp_event_type_t event_type = __odp_event_type_get(event); *subtype = event_type == ODP_EVENT_PACKET ? - odp_packet_subtype(odp_packet_from_event(event)) : + (odp_event_subtype_t)_odp_pkt_get((odp_packet_t)event, int8_t, subtype) : ODP_EVENT_NO_SUBTYPE; return event_type; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index b00173aca..2ae0829c3 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -51,6 +51,7 @@ typedef struct _odp_packet_inline_offset_t { uint16_t input_flags; uint16_t flags; uint16_t subtype; + uint16_t cls_mark; } _odp_packet_inline_offset_t; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index da4eabe2e..21d184ac5 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -14,16 +14,13 @@ #ifndef _ODP_PLAT_PACKET_INLINES_H_ #define _ODP_PLAT_PACKET_INLINES_H_ +#include <odp/api/event.h> #include <odp/api/hints.h> #include <odp/api/packet_types.h> #include <odp/api/pool_types.h> #include <odp/api/time.h> -#include <odp/api/abi/buffer.h> -#include <odp/api/abi/event_types.h> -#include <odp/api/abi/packet.h> -#include <odp/api/abi/packet_io.h> - +#include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/packet_io_inlines.h> #include <odp/api/plat/packet_inline_types.h> #include <odp/api/plat/pool_inline_types.h> @@ -48,6 +45,7 @@ #define odp_packet_input_index __odp_packet_input_index #define odp_packet_num_segs __odp_packet_num_segs #define odp_packet_user_ptr __odp_packet_user_ptr + #define odp_packet_user_ptr_set __odp_packet_user_ptr_set #define odp_packet_user_area __odp_packet_user_area #define odp_packet_user_area_size __odp_packet_user_area_size #define odp_packet_user_flag __odp_packet_user_flag @@ -55,11 +53,23 @@ #define odp_packet_l2_offset __odp_packet_l2_offset #define odp_packet_l3_offset __odp_packet_l3_offset #define odp_packet_l4_offset __odp_packet_l4_offset + #define odp_packet_l2_offset_set __odp_packet_l2_offset_set + #define odp_packet_l3_offset_set __odp_packet_l3_offset_set + #define odp_packet_l4_offset_set __odp_packet_l4_offset_set #define odp_packet_l2_ptr __odp_packet_l2_ptr #define odp_packet_l3_ptr __odp_packet_l3_ptr #define odp_packet_l4_ptr __odp_packet_l4_ptr + #define odp_packet_l2_type __odp_packet_l2_type + #define odp_packet_l3_type __odp_packet_l3_type + #define odp_packet_l4_type __odp_packet_l4_type + #define odp_packet_l3_chksum_status __odp_packet_l3_chksum_status + #define odp_packet_l4_chksum_status __odp_packet_l4_chksum_status + #define odp_packet_l3_chksum_insert __odp_packet_l3_chksum_insert + #define odp_packet_l4_chksum_insert __odp_packet_l4_chksum_insert #define odp_packet_flow_hash __odp_packet_flow_hash #define odp_packet_ts __odp_packet_ts + #define odp_packet_ts_set __odp_packet_ts_set + #define odp_packet_ts_request __odp_packet_ts_request #define odp_packet_head __odp_packet_head #define odp_packet_is_segmented __odp_packet_is_segmented #define odp_packet_first_seg __odp_packet_first_seg @@ -79,6 +89,7 @@ #define odp_packet_color __odp_packet_color #define odp_packet_drop_eligible __odp_packet_drop_eligible #define odp_packet_shaper_len_adjust __odp_packet_shaper_len_adjust + #define odp_packet_cls_mark __odp_packet_cls_mark #define odp_packet_buf_data_len __odp_packet_buf_data_len #define odp_packet_buf_size __odp_packet_buf_size #define odp_packet_buf_head __odp_packet_buf_head @@ -176,6 +187,20 @@ _ODP_INLINE void *odp_packet_user_ptr(odp_packet_t pkt) return _odp_pkt_get(pkt, void *, user_ptr); } +_ODP_INLINE void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ptr) +{ + _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + const void **user_ptr = _odp_pkt_get_ptr(pkt, const void *, user_ptr); + + if (odp_unlikely(ptr == NULL)) { + flags->user_ptr_set = 0; + return; + } + + *user_ptr = ptr; + flags->user_ptr_set = 1; +} + _ODP_INLINE void *odp_packet_user_area(odp_packet_t pkt) { return _odp_pkt_get(pkt, void *, user_area); @@ -219,6 +244,42 @@ _ODP_INLINE uint32_t odp_packet_l4_offset(odp_packet_t pkt) return _odp_pkt_get(pkt, uint16_t, l4_offset); } +_ODP_INLINE int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset) +{ + uint16_t *l2_offset = _odp_pkt_get_ptr(pkt, uint16_t, l2_offset); + _odp_packet_input_flags_t *input_flags = _odp_pkt_get_ptr(pkt, _odp_packet_input_flags_t, + input_flags); + + if (odp_unlikely(offset >= odp_packet_len(pkt))) + return -1; + + input_flags->l2 = 1; + *l2_offset = (uint16_t)offset; + return 0; +} + +_ODP_INLINE int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset) +{ + uint16_t *l3_offset = _odp_pkt_get_ptr(pkt, uint16_t, l3_offset); + + if (odp_unlikely(offset >= odp_packet_len(pkt))) + return -1; + + *l3_offset = (uint16_t)offset; + return 0; +} + +_ODP_INLINE int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset) +{ + uint16_t *l4_offset = _odp_pkt_get_ptr(pkt, uint16_t, l4_offset); + + if (odp_unlikely(offset >= odp_packet_len(pkt))) + return -1; + + *l4_offset = (uint16_t)offset; + return 0; +} + _ODP_INLINE void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len) { uint32_t offset = odp_packet_l2_offset(pkt); @@ -273,6 +334,107 @@ _ODP_INLINE void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len) return data + offset; } +_ODP_INLINE odp_proto_l2_type_t odp_packet_l2_type(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + return input_flags.eth ? ODP_PROTO_L2_TYPE_ETH : ODP_PROTO_L2_TYPE_NONE; +} + +_ODP_INLINE odp_proto_l3_type_t odp_packet_l3_type(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + if (input_flags.ipv4) + return ODP_PROTO_L3_TYPE_IPV4; + else if (input_flags.ipv6) + return ODP_PROTO_L3_TYPE_IPV6; + else if (input_flags.arp) + return ODP_PROTO_L3_TYPE_ARP; + + return ODP_PROTO_L3_TYPE_NONE; +} + +_ODP_INLINE odp_proto_l4_type_t odp_packet_l4_type(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + if (input_flags.tcp) + return ODP_PROTO_L4_TYPE_TCP; + else if (input_flags.udp) + return ODP_PROTO_L4_TYPE_UDP; + else if (input_flags.sctp) + return ODP_PROTO_L4_TYPE_SCTP; + else if (input_flags.ipsec_ah) + return ODP_PROTO_L4_TYPE_AH; + else if (input_flags.ipsec_esp) + return ODP_PROTO_L4_TYPE_ESP; + else if (input_flags.icmp && input_flags.ipv4) + return ODP_PROTO_L4_TYPE_ICMPV4; + else if (input_flags.icmp && input_flags.ipv6) + return ODP_PROTO_L4_TYPE_ICMPV6; + else if (input_flags.no_next_hdr) + return ODP_PROTO_L4_TYPE_NO_NEXT; + + return ODP_PROTO_L4_TYPE_NONE; +} + +_ODP_INLINE odp_packet_chksum_status_t odp_packet_l3_chksum_status(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + _odp_packet_input_flags_t input_flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + if (!input_flags.l3_chksum_done) + return ODP_PACKET_CHKSUM_UNKNOWN; + + if (flags.l3_chksum_err) + return ODP_PACKET_CHKSUM_BAD; + + return ODP_PACKET_CHKSUM_OK; +} + +_ODP_INLINE odp_packet_chksum_status_t odp_packet_l4_chksum_status(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + _odp_packet_input_flags_t input_flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + if (!input_flags.l4_chksum_done) + return ODP_PACKET_CHKSUM_UNKNOWN; + + if (flags.l4_chksum_err) + return ODP_PACKET_CHKSUM_BAD; + + return ODP_PACKET_CHKSUM_OK; +} + +_ODP_INLINE void odp_packet_l3_chksum_insert(odp_packet_t pkt, int insert) +{ + _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + + flags->l3_chksum_set = 1; + flags->l3_chksum = !!insert; +} + +_ODP_INLINE void odp_packet_l4_chksum_insert(odp_packet_t pkt, int insert) +{ + _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + + flags->l4_chksum_set = 1; + flags->l4_chksum = !!insert; +} + _ODP_INLINE uint32_t odp_packet_flow_hash(odp_packet_t pkt) { return _odp_pkt_get(pkt, uint32_t, flow_hash); @@ -283,6 +445,23 @@ _ODP_INLINE odp_time_t odp_packet_ts(odp_packet_t pkt) return _odp_pkt_get(pkt, odp_time_t, timestamp); } +_ODP_INLINE void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) +{ + odp_time_t *ts = _odp_pkt_get_ptr(pkt, odp_time_t, timestamp); + _odp_packet_input_flags_t *input_flags = _odp_pkt_get_ptr(pkt, _odp_packet_input_flags_t, + input_flags); + + *ts = timestamp; + input_flags->timestamp = 1; +} + +_ODP_INLINE void odp_packet_ts_request(odp_packet_t pkt, int enable) +{ + _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + + flags->ts_set = !!enable; +} + _ODP_INLINE void *odp_packet_head(odp_packet_t pkt) { return (uint8_t *)odp_packet_data(pkt) - odp_packet_headroom(pkt); @@ -364,6 +543,8 @@ _ODP_INLINE int odp_packet_copy_to_mem(odp_packet_t pkt, uint32_t offset, _ODP_INLINE odp_packet_t odp_packet_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); + return (odp_packet_t)ev; } @@ -398,6 +579,8 @@ _ODP_INLINE odp_event_subtype_t odp_packet_subtype(odp_packet_t pkt) _ODP_INLINE odp_packet_tx_compl_t odp_packet_tx_compl_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET_TX_COMPL); + return (odp_packet_tx_compl_t)(uintptr_t)ev; } @@ -433,6 +616,15 @@ _ODP_INLINE int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt) return (int8_t)flags.shaper_len_adj; } +_ODP_INLINE uint64_t odp_packet_cls_mark(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + return input_flags.cls_mark ? _odp_pkt_get(pkt, uint16_t, cls_mark) : 0; +} + _ODP_INLINE uint32_t odp_packet_buf_data_len(odp_packet_buf_t pkt_buf) { return _odp_pkt_get(pkt_buf, uint32_t, seg_len); diff --git a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h index 76604dc4f..08fb07a62 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h @@ -14,10 +14,11 @@ #ifndef _ODP_PLAT_PACKET_VECTOR_INLINES_H_ #define _ODP_PLAT_PACKET_VECTOR_INLINES_H_ -#include <odp/api/event_types.h> +#include <odp/api/event.h> #include <odp/api/packet_types.h> #include <odp/api/pool_types.h> +#include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/event_vector_inline_types.h> #include <stdint.h> @@ -45,6 +46,8 @@ extern const _odp_event_vector_inline_offset_t _odp_event_vector_inline; _ODP_INLINE odp_packet_vector_t odp_packet_vector_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET_VECTOR); + return (odp_packet_vector_t)ev; } diff --git a/platform/linux-generic/include/odp/api/plat/rwlock_inlines.h b/platform/linux-generic/include/odp/api/plat/rwlock_inlines.h new file mode 100644 index 000000000..0bb9d8c6e --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/rwlock_inlines.h @@ -0,0 +1,105 @@ +/* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_RWLOCK_INLINES_H_ +#define ODP_PLAT_RWLOCK_INLINES_H_ + +#include <odp/api/atomic.h> +#include <odp/api/cpu.h> + +#include <odp/api/abi/rwlock.h> + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_rwlock_init __odp_rwlock_init + #define odp_rwlock_read_lock __odp_rwlock_read_lock + #define odp_rwlock_read_trylock __odp_rwlock_read_trylock + #define odp_rwlock_read_unlock __odp_rwlock_read_unlock + #define odp_rwlock_write_lock __odp_rwlock_write_lock + #define odp_rwlock_write_trylock __odp_rwlock_write_trylock + #define odp_rwlock_write_unlock __odp_rwlock_write_unlock +#else + #undef _ODP_INLINE + #define _ODP_INLINE +#endif + +_ODP_INLINE void odp_rwlock_init(odp_rwlock_t *rwlock) +{ + odp_atomic_init_u32(&rwlock->cnt, 0); +} + +_ODP_INLINE void odp_rwlock_read_lock(odp_rwlock_t *rwlock) +{ + uint32_t cnt; + int is_locked = 0; + + while (is_locked == 0) { + cnt = odp_atomic_load_u32(&rwlock->cnt); + /* waiting for read lock */ + if ((int32_t)cnt < 0) { + odp_cpu_pause(); + continue; + } + is_locked = odp_atomic_cas_acq_u32(&rwlock->cnt, &cnt, cnt + 1); + } +} + +_ODP_INLINE int odp_rwlock_read_trylock(odp_rwlock_t *rwlock) +{ + uint32_t cnt = odp_atomic_load_u32(&rwlock->cnt); + + while (cnt != (uint32_t)-1) { + if (odp_atomic_cas_acq_u32(&rwlock->cnt, &cnt, cnt + 1)) + return 1; + } + + return 0; +} + +_ODP_INLINE void odp_rwlock_read_unlock(odp_rwlock_t *rwlock) +{ + odp_atomic_sub_rel_u32(&rwlock->cnt, 1); +} + +_ODP_INLINE void odp_rwlock_write_lock(odp_rwlock_t *rwlock) +{ + uint32_t cnt; + int is_locked = 0; + + while (is_locked == 0) { + uint32_t zero = 0; + + cnt = odp_atomic_load_u32(&rwlock->cnt); + /* lock acquired, wait */ + if (cnt != 0) { + odp_cpu_pause(); + continue; + } + is_locked = odp_atomic_cas_acq_u32(&rwlock->cnt, &zero, (uint32_t)-1); + } +} + +_ODP_INLINE int odp_rwlock_write_trylock(odp_rwlock_t *rwlock) +{ + uint32_t zero = 0; + + return odp_atomic_cas_acq_u32(&rwlock->cnt, &zero, (uint32_t)-1); +} + +_ODP_INLINE void odp_rwlock_write_unlock(odp_rwlock_t *rwlock) +{ + odp_atomic_store_rel_u32(&rwlock->cnt, 0); +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/rwlock_recursive_inlines.h b/platform/linux-generic/include/odp/api/plat/rwlock_recursive_inlines.h new file mode 100644 index 000000000..21ad4be4a --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/rwlock_recursive_inlines.h @@ -0,0 +1,142 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_RWLOCK_RECURSIVE_INLINES_H_ +#define ODP_PLAT_RWLOCK_RECURSIVE_INLINES_H_ + +#include <odp/api/rwlock.h> +#include <odp/api/thread.h> + +#include <odp/api/abi/rwlock_recursive.h> + +#include <odp/api/plat/debug_inlines.h> + +#include <stdint.h> +#include <string.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_rwlock_recursive_init __odp_rwlock_recursive_init + #define odp_rwlock_recursive_read_lock __odp_rwlock_recursive_read_lock + #define odp_rwlock_recursive_read_trylock __odp_rwlock_recursive_read_trylock + #define odp_rwlock_recursive_read_unlock __odp_rwlock_recursive_read_unlock + #define odp_rwlock_recursive_write_lock __odp_rwlock_recursive_write_lock + #define odp_rwlock_recursive_write_trylock __odp_rwlock_recursive_write_trylock + #define odp_rwlock_recursive_write_unlock __odp_rwlock_recursive_write_unlock +#else + #undef _ODP_INLINE + #define _ODP_INLINE +#endif + +_ODP_INLINE void odp_rwlock_recursive_init(odp_rwlock_recursive_t *rlock) +{ + memset(rlock, 0, sizeof(odp_rwlock_recursive_t)); + odp_rwlock_init(&rlock->lock); + rlock->wr_owner = -1; +} + +/* Multiple readers can recurse the lock concurrently */ +_ODP_INLINE void odp_rwlock_recursive_read_lock(odp_rwlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->rd_cnt[thr]) { + _ODP_ASSERT(rlock->rd_cnt[thr] < UINT8_MAX); + rlock->rd_cnt[thr]++; + return; + } + + odp_rwlock_read_lock(&rlock->lock); + rlock->rd_cnt[thr] = 1; +} + +/* Multiple readers can recurse the lock concurrently */ +_ODP_INLINE int odp_rwlock_recursive_read_trylock(odp_rwlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->rd_cnt[thr]) { + _ODP_ASSERT(rlock->rd_cnt[thr] < UINT8_MAX); + rlock->rd_cnt[thr]++; + return 1; + } + + if (odp_rwlock_read_trylock(&rlock->lock)) { + rlock->rd_cnt[thr] = 1; + return 1; + } + + return 0; +} + +_ODP_INLINE void odp_rwlock_recursive_read_unlock(odp_rwlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + _ODP_ASSERT(rlock->rd_cnt[thr]); + rlock->rd_cnt[thr]--; + + if (rlock->rd_cnt[thr] > 0) + return; + + odp_rwlock_read_unlock(&rlock->lock); +} + +/* Only one writer can recurse the lock */ +_ODP_INLINE void odp_rwlock_recursive_write_lock(odp_rwlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->wr_owner == thr) { + _ODP_ASSERT(rlock->wr_cnt < UINT32_MAX); + rlock->wr_cnt++; + return; + } + + odp_rwlock_write_lock(&rlock->lock); + rlock->wr_owner = thr; + rlock->wr_cnt = 1; +} + +/* Only one writer can recurse the lock */ +_ODP_INLINE int odp_rwlock_recursive_write_trylock(odp_rwlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->wr_owner == thr) { + _ODP_ASSERT(rlock->wr_cnt < UINT32_MAX); + rlock->wr_cnt++; + return 1; + } + + if (odp_rwlock_write_trylock(&rlock->lock)) { + rlock->wr_owner = thr; + rlock->wr_cnt = 1; + return 1; + } + + return 0; +} + +_ODP_INLINE void odp_rwlock_recursive_write_unlock(odp_rwlock_recursive_t *rlock) +{ + _ODP_ASSERT(rlock->wr_cnt); + rlock->wr_cnt--; + + if (rlock->wr_cnt > 0) + return; + + rlock->wr_owner = -1; + odp_rwlock_write_unlock(&rlock->lock); +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/schedule_inline_types.h b/platform/linux-generic/include/odp/api/plat/schedule_inline_types.h new file mode 100644 index 000000000..92089c9a0 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/schedule_inline_types.h @@ -0,0 +1,68 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_SCHEDULE_INLINE_TYPES_H_ +#define ODP_PLAT_SCHEDULE_INLINE_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/event_types.h> +#include <odp/api/queue_types.h> +#include <odp/api/schedule_types.h> +#include <odp/api/thrmask.h> + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +/* Schedule API functions */ +typedef struct { + uint64_t (*schedule_wait_time)(uint64_t ns); + int (*schedule_capability)(odp_schedule_capability_t *capa); + void (*schedule_config_init)(odp_schedule_config_t *config); + int (*schedule_config)(const odp_schedule_config_t *config); + odp_event_t (*schedule)(odp_queue_t *from, uint64_t wait); + int (*schedule_multi)(odp_queue_t *from, uint64_t wait, odp_event_t events[], int num); + int (*schedule_multi_wait)(odp_queue_t *from, odp_event_t events[], int num); + int (*schedule_multi_no_wait)(odp_queue_t *from, odp_event_t events[], int num); + void (*schedule_pause)(void); + void (*schedule_resume)(void); + void (*schedule_release_atomic)(void); + void (*schedule_release_ordered)(void); + void (*schedule_prefetch)(int num); + int (*schedule_min_prio)(void); + int (*schedule_max_prio)(void); + int (*schedule_default_prio)(void); + int (*schedule_num_prio)(void); + odp_schedule_group_t (*schedule_group_create)(const char *name, const odp_thrmask_t *mask); + int (*schedule_group_destroy)(odp_schedule_group_t group); + odp_schedule_group_t (*schedule_group_lookup)(const char *name); + int (*schedule_group_join)(odp_schedule_group_t group, const odp_thrmask_t *mask); + int (*schedule_group_leave)(odp_schedule_group_t group, const odp_thrmask_t *mask); + int (*schedule_group_thrmask)(odp_schedule_group_t group, odp_thrmask_t *mask); + int (*schedule_group_info)(odp_schedule_group_t group, odp_schedule_group_info_t *info); + void (*schedule_order_lock)(uint32_t lock_index); + void (*schedule_order_unlock)(uint32_t lock_index); + void (*schedule_order_unlock_lock)(uint32_t unlock_index, uint32_t lock_index); + void (*schedule_order_lock_start)(uint32_t lock_index); + void (*schedule_order_lock_wait)(uint32_t lock_index); + void (*schedule_order_wait)(void); + void (*schedule_print)(void); + +} _odp_schedule_api_fn_t; + +/* Scheduler configuration status */ +int _odp_schedule_configured(void); + +/** @endcond */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/schedule_inlines.h b/platform/linux-generic/include/odp/api/plat/schedule_inlines.h new file mode 100644 index 000000000..733b068d9 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/schedule_inlines.h @@ -0,0 +1,135 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_SCHEDULE_INLINES_H_ +#define ODP_PLAT_SCHEDULE_INLINES_H_ + +#include <odp/api/event_types.h> +#include <odp/api/queue_types.h> + +#include <odp/api/plat/debug_inlines.h> +#include <odp/api/plat/schedule_inline_types.h> + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +extern const _odp_schedule_api_fn_t *_odp_sched_api; + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_schedule __odp_schedule + #define odp_schedule_multi __odp_schedule_multi + #define odp_schedule_multi_wait __odp_schedule_multi_wait + #define odp_schedule_multi_no_wait __odp_schedule_multi_no_wait + #define odp_schedule_wait_time __odp_schedule_wait_time + #define odp_schedule_pause __odp_schedule_pause + #define odp_schedule_resume __odp_schedule_resume + #define odp_schedule_release_atomic __odp_schedule_release_atomic + #define odp_schedule_release_ordered __odp_schedule_release_ordered + #define odp_schedule_prefetch __odp_schedule_prefetch + #define odp_schedule_order_lock __odp_schedule_order_lock + #define odp_schedule_order_unlock __odp_schedule_order_unlock + #define odp_schedule_order_unlock_lock __odp_schedule_order_unlock_lock + #define odp_schedule_order_lock_start __odp_schedule_order_lock_start + #define odp_schedule_order_lock_wait __odp_schedule_order_lock_wait + #define odp_schedule_order_wait __odp_schedule_order_wait +#else + #define _ODP_INLINE +#endif + +_ODP_INLINE odp_event_t odp_schedule(odp_queue_t *from, uint64_t wait) +{ + _ODP_ASSERT(_odp_schedule_configured()); + + return _odp_sched_api->schedule(from, wait); +} + +_ODP_INLINE int odp_schedule_multi(odp_queue_t *from, uint64_t wait, odp_event_t events[], int num) +{ + _ODP_ASSERT(_odp_schedule_configured()); + + return _odp_sched_api->schedule_multi(from, wait, events, num); +} + +_ODP_INLINE int odp_schedule_multi_wait(odp_queue_t *from, odp_event_t events[], int num) +{ + _ODP_ASSERT(_odp_schedule_configured()); + + return _odp_sched_api->schedule_multi_wait(from, events, num); +} + +_ODP_INLINE int odp_schedule_multi_no_wait(odp_queue_t *from, odp_event_t events[], int num) +{ + _ODP_ASSERT(_odp_schedule_configured()); + + return _odp_sched_api->schedule_multi_no_wait(from, events, num); +} + +_ODP_INLINE uint64_t odp_schedule_wait_time(uint64_t ns) +{ + return _odp_sched_api->schedule_wait_time(ns); +} + +_ODP_INLINE void odp_schedule_pause(void) +{ + _odp_sched_api->schedule_pause(); +} + +_ODP_INLINE void odp_schedule_resume(void) +{ + _odp_sched_api->schedule_resume(); +} + +_ODP_INLINE void odp_schedule_release_atomic(void) +{ + _odp_sched_api->schedule_release_atomic(); +} + +_ODP_INLINE void odp_schedule_release_ordered(void) +{ + _odp_sched_api->schedule_release_ordered(); +} + +_ODP_INLINE void odp_schedule_prefetch(int num) +{ + _odp_sched_api->schedule_prefetch(num); +} + +_ODP_INLINE void odp_schedule_order_lock(uint32_t lock_index) +{ + _odp_sched_api->schedule_order_lock(lock_index); +} + +_ODP_INLINE void odp_schedule_order_unlock(uint32_t lock_index) +{ + _odp_sched_api->schedule_order_unlock(lock_index); +} + +_ODP_INLINE void odp_schedule_order_unlock_lock(uint32_t unlock_index, uint32_t lock_index) +{ + _odp_sched_api->schedule_order_unlock_lock(unlock_index, lock_index); +} + +_ODP_INLINE void odp_schedule_order_lock_start(uint32_t lock_index) +{ + _odp_sched_api->schedule_order_lock_start(lock_index); +} + +_ODP_INLINE void odp_schedule_order_lock_wait(uint32_t lock_index) +{ + _odp_sched_api->schedule_order_lock_wait(lock_index); +} + +_ODP_INLINE void odp_schedule_order_wait(void) +{ + _odp_sched_api->schedule_order_wait(); +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h b/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h index 2dd846fe9..e795353f4 100644 --- a/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h @@ -13,6 +13,10 @@ #include <odp/api/abi/spinlock_recursive.h> +#include <odp/api/plat/debug_inlines.h> + +#include <stdint.h> + /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ #ifndef _ODP_NO_INLINE @@ -43,6 +47,7 @@ _ODP_INLINE void odp_spinlock_recursive_lock(odp_spinlock_recursive_t *rlock) int thr = odp_thread_id(); if (rlock->owner == thr) { + _ODP_ASSERT(rlock->cnt < UINT32_MAX); rlock->cnt++; return; } @@ -57,6 +62,7 @@ _ODP_INLINE int odp_spinlock_recursive_trylock(odp_spinlock_recursive_t *rlock) int thr = odp_thread_id(); if (rlock->owner == thr) { + _ODP_ASSERT(rlock->cnt < UINT32_MAX); rlock->cnt++; return 1; } @@ -72,6 +78,7 @@ _ODP_INLINE int odp_spinlock_recursive_trylock(odp_spinlock_recursive_t *rlock) _ODP_INLINE void odp_spinlock_recursive_unlock(odp_spinlock_recursive_t *rlock) { + _ODP_ASSERT(rlock->cnt); rlock->cnt--; if (rlock->cnt > 0) diff --git a/platform/linux-generic/include/odp/api/plat/thread_inline_types.h b/platform/linux-generic/include/odp/api/plat/thread_inline_types.h new file mode 100644 index 000000000..c9a15b06b --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/thread_inline_types.h @@ -0,0 +1,36 @@ +/* Copyright (c) 2018-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_THREAD_INLINE_TYPES_H_ +#define ODP_PLAT_THREAD_INLINE_TYPES_H_ + +#include <odp/api/init.h> +#include <odp/api/spec/thread_types.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +typedef struct { + odp_log_func_t log_fn; + odp_thread_type_t type; + int thr; + int cpu; + +} _odp_thread_state_t; + +extern __thread _odp_thread_state_t *_odp_this_thread; + +/** @endcond */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/thread_inlines.h b/platform/linux-generic/include/odp/api/plat/thread_inlines.h index fc9275209..2f2e50b06 100644 --- a/platform/linux-generic/include/odp/api/plat/thread_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/thread_inlines.h @@ -9,22 +9,14 @@ #include <odp/api/init.h> +#include <odp/api/plat/thread_inline_types.h> + #ifdef __cplusplus extern "C" { #endif /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -typedef struct { - int thr; - int cpu; - odp_thread_type_t type; - odp_log_func_t log_fn; - -} _odp_thread_state_t; - -extern __thread _odp_thread_state_t *_odp_this_thread; - #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline diff --git a/platform/linux-generic/include/odp/api/plat/timer_inlines.h b/platform/linux-generic/include/odp/api/plat/timer_inlines.h index 7642376d0..4f5620caf 100644 --- a/platform/linux-generic/include/odp/api/plat/timer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/timer_inlines.h @@ -10,6 +10,7 @@ #include <odp/api/event.h> #include <odp/api/timer_types.h> +#include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/timer_inline_types.h> #include <stdint.h> @@ -71,6 +72,8 @@ _ODP_INLINE uint64_t odp_timer_ns_to_tick(odp_timer_pool_t tp, uint64_t ns) _ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_TIMEOUT); + return (odp_timeout_t)ev; } diff --git a/platform/linux-generic/include/odp_classification_datamodel.h b/platform/linux-generic/include/odp_classification_datamodel.h index 213db03db..00df57bef 100644 --- a/platform/linux-generic/include/odp_classification_datamodel.h +++ b/platform/linux-generic/include/odp_classification_datamodel.h @@ -132,24 +132,24 @@ typedef struct pmr_term_value { Class Of Service */ typedef struct ODP_ALIGNED_CACHE cos_s { + uint32_t valid; /* validity Flag */ + odp_atomic_u32_t num_rule; /* num of PMRs attached with this CoS */ + struct pmr_s *pmr[CLS_PMR_PER_COS_MAX]; /* Chained PMR */ + struct cos_s *linked_cos[CLS_PMR_PER_COS_MAX]; /* Chained CoS with PMR*/ + odp_bool_t stats_enable; odp_cos_action_t action; /* Action */ odp_queue_t queue; /* Associated Queue */ + uint32_t num_queue; odp_pool_t pool; /* Associated Buffer pool */ + uint8_t index; + bool queue_group; + odp_cls_hash_proto_t hash_proto; odp_pktin_vector_config_t vector; /* Packet vector config */ - struct pmr_s *pmr[CLS_PMR_PER_COS_MAX]; /* Chained PMR */ - struct cos_s *linked_cos[CLS_PMR_PER_COS_MAX]; /* Chained CoS with PMR*/ - uint32_t valid; /* validity Flag */ odp_cls_drop_t drop_policy; /* Associated Drop Policy */ size_t headroom; /* Headroom for this CoS */ odp_spinlock_t lock; /* cos lock */ - odp_atomic_u32_t num_rule; /* num of PMRs attached with this CoS */ - bool queue_group; - odp_cls_hash_proto_t hash_proto; - uint32_t num_queue; odp_queue_param_t queue_param; char name[ODP_COS_NAME_LEN]; /* name */ - uint8_t index; - odp_bool_t stats_enable; struct { odp_atomic_u64_t discards; odp_atomic_u64_t packets; @@ -159,13 +159,12 @@ typedef struct ODP_ALIGNED_CACHE cos_s { /* Pattern Matching Rule */ typedef struct ODP_ALIGNED_CACHE pmr_s { uint32_t valid; /* Validity Flag */ - odp_atomic_u32_t count; /* num of packets matching this rule */ uint32_t num_pmr; /* num of PMR Term Values*/ uint16_t mark; - odp_spinlock_t lock; /* pmr lock*/ - cos_t *src_cos; /* source CoS where PMR is attached */ pmr_term_value_t pmr_term_value[CLS_PMRTERM_MAX]; /* List of associated PMR Terms */ + odp_spinlock_t lock; /* pmr lock*/ + cos_t *src_cos; /* source CoS where PMR is attached */ } pmr_t; typedef struct ODP_ALIGNED_CACHE { diff --git a/platform/linux-generic/include/odp_classification_internal.h b/platform/linux-generic/include/odp_classification_internal.h index dd1f8aaf6..c3ecf4079 100644 --- a/platform/linux-generic/include/odp_classification_internal.h +++ b/platform/linux-generic/include/odp_classification_internal.h @@ -62,7 +62,7 @@ static inline void _odp_cos_queue_stats_add(cos_t *cos, odp_queue_t queue, int queue_idx = _odp_cos_queue_idx(cos, queue); if (odp_unlikely(queue_idx < 0)) { - ODP_ERR("Queue not attached to the CoS\n"); + _ODP_ERR("Queue not attached to the CoS\n"); return; } diff --git a/platform/linux-generic/include/odp_debug_internal.h b/platform/linux-generic/include/odp_debug_internal.h index 22dca3701..4184d5170 100644 --- a/platform/linux-generic/include/odp_debug_internal.h +++ b/platform/linux-generic/include/odp_debug_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2020, Nokia + * Copyright (c) 2020-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -18,9 +18,10 @@ #define ODP_DEBUG_INTERNAL_H_ #include <odp/autoheader_external.h> + #include <odp/api/debug.h> -#include <odp_global_data.h> -#include <odp/api/plat/thread_inlines.h> + +#include <odp/api/plat/debug_inlines.h> #include <stdio.h> #include <stdlib.h> @@ -37,23 +38,6 @@ extern "C" { * level 0 to N. */ #define CONFIG_DEBUG_LEVEL 0 -#define _ODP_LOG_FN(level, fmt, ...) \ - do { \ - if (_odp_this_thread && _odp_this_thread->log_fn) \ - _odp_this_thread->log_fn(level, fmt, ##__VA_ARGS__); \ - else \ - odp_global_ro.log_fn(level, fmt, ##__VA_ARGS__); \ - } while (0) - -/** - * Runtime assertion-macro - aborts if 'cond' is false. - */ -#define ODP_ASSERT(cond) \ - do { if ((ODP_DEBUG == 1) && (!(cond))) { \ - ODP_ERR("%s\n", #cond); \ - odp_global_ro.abort_fn(); } \ - } while (0) - /** * This macro is used to indicate when a given function is not implemented */ @@ -61,14 +45,6 @@ extern "C" { _ODP_LOG_FN(ODP_LOG_UNIMPLEMENTED, \ "%s:%d:The function %s() is not implemented\n", \ __FILE__, __LINE__, __func__) -/* - * Print debug message to log, if ODP_DEBUG_PRINT flag is set (ignores CONFIG_DEBUG_LEVEL). - */ -#define ODP_DBG(fmt, ...) \ - do { \ - if (ODP_DEBUG_PRINT == 1) \ - ODP_LOG(ODP_LOG_DBG, fmt, ##__VA_ARGS__);\ - } while (0) /* * Print debug message to log, if ODP_DEBUG_PRINT flag is set and CONFIG_DEBUG_LEVEL is high enough. @@ -76,7 +52,7 @@ extern "C" { #define ODP_DBG_LVL(level, fmt, ...) \ do { \ if (ODP_DEBUG_PRINT == 1 && CONFIG_DEBUG_LEVEL >= (level)) \ - ODP_LOG(ODP_LOG_DBG, fmt, ##__VA_ARGS__);\ + _ODP_LOG(ODP_LOG_DBG, fmt, ##__VA_ARGS__);\ } while (0) /* @@ -88,36 +64,6 @@ extern "C" { _ODP_LOG_FN(ODP_LOG_DBG, fmt, ##__VA_ARGS__);\ } while (0) -/** - * Log error message. - */ -#define ODP_ERR(fmt, ...) \ - ODP_LOG(ODP_LOG_ERR, fmt, ##__VA_ARGS__) - -/** - * Log abort message and then stop execution (by default call abort()). - * This function should not return. - */ -#define ODP_ABORT(fmt, ...) \ - do { \ - ODP_LOG(ODP_LOG_ABORT, fmt, ##__VA_ARGS__); \ - odp_global_ro.abort_fn(); \ - } while (0) - -/** - * ODP LOG macro. - */ -#define ODP_LOG(level, fmt, ...) \ - _ODP_LOG_FN(level, "%s:%d:%s():" fmt, __FILE__, \ - __LINE__, __func__, ##__VA_ARGS__) - -/** - * Log print message when the application calls one of the ODP APIs - * specifically for dumping internal data. - */ -#define ODP_PRINT(fmt, ...) \ - _ODP_LOG_FN(ODP_LOG_PRINT, fmt, ##__VA_ARGS__) - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index 462b8d639..d4cc9cda4 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -61,8 +61,6 @@ typedef struct odp_global_data_ro_t { pid_t main_pid; pid_t fdserver_pid; char uid[UID_MAXLEN]; - odp_log_func_t ODP_PRINTF_FORMAT(2, 3) log_fn; - odp_abort_func_t abort_fn; system_info_t system_info; hugepage_info_t hugepage_info; odp_cpumask_t all_cpus; diff --git a/platform/linux-generic/include/odp_llqueue.h b/platform/linux-generic/include/odp_llqueue.h index 6340d111a..29810ebf3 100644 --- a/platform/linux-generic/include/odp_llqueue.h +++ b/platform/linux-generic/include/odp_llqueue.h @@ -84,7 +84,7 @@ static inline void llq_enqueue(struct llqueue *llq, struct llnode *node) { union llht old, neu; - ODP_ASSERT(node->next == NULL); + _ODP_ASSERT(node->next == NULL); node->next = SENTINEL; do { old.ui = lld(&llq->u.ui, __ATOMIC_RELAXED); @@ -93,7 +93,7 @@ static inline void llq_enqueue(struct llqueue *llq, struct llnode *node) } while (odp_unlikely(scd(&llq->u.ui, neu.ui, __ATOMIC_RELEASE))); if (old.st.tail != NULL) { /* List was not empty */ - ODP_ASSERT(old.st.tail->next == SENTINEL); + _ODP_ASSERT(old.st.tail->next == SENTINEL); old.st.tail->next = node; } } @@ -230,7 +230,7 @@ static inline void llqueue_init(struct llqueue *llq) static inline void llq_enqueue(struct llqueue *llq, struct llnode *node) { - ODP_ASSERT(node->next == NULL); + _ODP_ASSERT(node->next == NULL); node->next = SENTINEL; odp_spinlock_lock(&llq->lock); @@ -257,11 +257,11 @@ static inline struct llnode *llq_dequeue(struct llqueue *llq) if (llq->head != NULL) { node = llq->head; if (llq->head == llq->tail) { - ODP_ASSERT(node->next == SENTINEL); + _ODP_ASSERT(node->next == SENTINEL); llq->head = NULL; llq->tail = NULL; } else { - ODP_ASSERT(node->next != SENTINEL); + _ODP_ASSERT(node->next != SENTINEL); llq->head = node->next; } node->next = NULL; @@ -279,11 +279,11 @@ static inline odp_bool_t llq_dequeue_cond(struct llqueue *llq, if (odp_likely(llq->head != NULL && llq->head == node)) { success = true; if (llq->head == llq->tail) { - ODP_ASSERT(node->next == SENTINEL); + _ODP_ASSERT(node->next == SENTINEL); llq->head = NULL; llq->tail = NULL; } else { - ODP_ASSERT(node->next != SENTINEL); + _ODP_ASSERT(node->next != SENTINEL); llq->head = node->next; } node->next = NULL; @@ -302,7 +302,7 @@ static inline odp_bool_t llq_cond_rotate(struct llqueue *llq, if (odp_likely(llq->head == node)) { success = true; if (llq->tail != node) { - ODP_ASSERT(node->next != SENTINEL); + _ODP_ASSERT(node->next != SENTINEL); llq->head = node->next; llq->tail->next = node; llq->tail = node; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 253650b6d..a6a8c551c 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -342,8 +342,8 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr, const uint32_t src_uarea_size = src_pool->param_uarea_size; const uint32_t dst_uarea_size = dst_pool->param_uarea_size; - ODP_ASSERT(dst_hdr->uarea_addr != NULL); - ODP_ASSERT(dst_uarea_size >= src_uarea_size); + _ODP_ASSERT(dst_hdr->uarea_addr != NULL); + _ODP_ASSERT(dst_uarea_size >= src_uarea_size); memcpy(dst_hdr->uarea_addr, src_hdr->uarea_addr, src_uarea_size); } else { @@ -351,7 +351,7 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr, /* If user area exists, packets should always be from the same pool, so * user area pointers can simply be swapped. */ - ODP_ASSERT(dst_hdr->event_hdr.pool == src_hdr->event_hdr.pool); + _ODP_ASSERT(dst_hdr->event_hdr.pool == src_hdr->event_hdr.pool); src_hdr->uarea_addr = dst_hdr->uarea_addr; dst_hdr->uarea_addr = src_uarea; @@ -440,11 +440,6 @@ static inline int packet_hdr_has_l2(odp_packet_hdr_t *pkt_hdr) return pkt_hdr->p.input_flags.l2; } -static inline void packet_hdr_has_l2_set(odp_packet_hdr_t *pkt_hdr, int val) -{ - pkt_hdr->p.input_flags.l2 = val; -} - static inline int packet_hdr_has_eth(odp_packet_hdr_t *pkt_hdr) { return pkt_hdr->p.input_flags.eth; diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index 04a7ea286..844088ac1 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -268,8 +268,8 @@ static inline pktio_entry_t *get_pktio_entry(odp_pktio_t pktio) return NULL; if (odp_unlikely(_odp_typeval(pktio) > ODP_CONFIG_PKTIO_ENTRIES)) { - ODP_DBG("pktio limit %" PRIuPTR "/%d exceed\n", - _odp_typeval(pktio), ODP_CONFIG_PKTIO_ENTRIES); + _ODP_DBG("pktio limit %" PRIuPTR "/%d exceed\n", + _odp_typeval(pktio), ODP_CONFIG_PKTIO_ENTRIES); return NULL; } diff --git a/platform/linux-generic/include/odp_parse_internal.h b/platform/linux-generic/include/odp_parse_internal.h index c467abbcd..8b3fb480b 100644 --- a/platform/linux-generic/include/odp_parse_internal.h +++ b/platform/linux-generic/include/odp_parse_internal.h @@ -59,10 +59,13 @@ uint16_t _odp_parse_eth(packet_parser_t *prs, const uint8_t **parseptr, * * See _odp_packet_parse_common(). Requires up to PARSE_L3_L4_BYTES bytes of * contiguous packet data. + * + * - offset is the offset of the first byte of the data pointed to by parseptr + * - seg_end is the maximum offset that can be accessed plus one */ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, uint32_t offset, - uint32_t frame_len, uint32_t seg_len, + uint32_t frame_len, uint32_t seg_end, int layer, uint16_t ethtype, uint64_t *l4_part_sum, odp_pktin_config_opt_t opt); diff --git a/platform/linux-generic/include/odp_queue_scalable_internal.h b/platform/linux-generic/include/odp_queue_scalable_internal.h index f7fc94e71..dccc2aef9 100644 --- a/platform/linux-generic/include/odp_queue_scalable_internal.h +++ b/platform/linux-generic/include/odp_queue_scalable_internal.h @@ -67,7 +67,7 @@ static inline void *shm_pool_alloc_align(_odp_ishm_pool_t *pool, uint32_t size) void *addr; addr = _odp_ishm_pool_alloc(pool, _ODP_ROUNDUP_CACHE_LINE(size)); - ODP_ASSERT(((uintptr_t)addr & (ODP_CACHE_LINE_SIZE - 1)) == 0); + _ODP_ASSERT(((uintptr_t)addr & (ODP_CACHE_LINE_SIZE - 1)) == 0); return addr; } diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index d9ba165bc..c70fdcf7c 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -91,51 +91,6 @@ int _odp_sched_cb_pktin_poll(int pktio_index, int pktin_index, _odp_event_hdr_t *hdr_tbl[], int num); void _odp_sched_cb_pktio_stop_finalize(int pktio_index); -/* API functions */ -typedef struct { - uint64_t (*schedule_wait_time)(uint64_t ns); - int (*schedule_capability)(odp_schedule_capability_t *capa); - void (*schedule_config_init)(odp_schedule_config_t *config); - int (*schedule_config)(const odp_schedule_config_t *config); - odp_event_t (*schedule)(odp_queue_t *from, uint64_t wait); - int (*schedule_multi)(odp_queue_t *from, uint64_t wait, - odp_event_t events[], int num); - int (*schedule_multi_wait)(odp_queue_t *from, odp_event_t events[], - int num); - int (*schedule_multi_no_wait)(odp_queue_t *from, odp_event_t events[], - int num); - void (*schedule_pause)(void); - void (*schedule_resume)(void); - void (*schedule_release_atomic)(void); - void (*schedule_release_ordered)(void); - void (*schedule_prefetch)(int num); - int (*schedule_min_prio)(void); - int (*schedule_max_prio)(void); - int (*schedule_default_prio)(void); - int (*schedule_num_prio)(void); - odp_schedule_group_t (*schedule_group_create) - (const char *name, const odp_thrmask_t *mask); - int (*schedule_group_destroy)(odp_schedule_group_t group); - odp_schedule_group_t (*schedule_group_lookup)(const char *name); - int (*schedule_group_join)(odp_schedule_group_t group, - const odp_thrmask_t *mask); - int (*schedule_group_leave)(odp_schedule_group_t group, - const odp_thrmask_t *mask); - int (*schedule_group_thrmask)(odp_schedule_group_t group, - odp_thrmask_t *mask); - int (*schedule_group_info)(odp_schedule_group_t group, - odp_schedule_group_info_t *info); - void (*schedule_order_lock)(uint32_t lock_index); - void (*schedule_order_unlock)(uint32_t lock_index); - void (*schedule_order_unlock_lock)(uint32_t unlock_index, - uint32_t lock_index); - void (*schedule_order_lock_start)(uint32_t lock_index); - void (*schedule_order_lock_wait)(uint32_t lock_index); - void (*schedule_order_wait)(void); - void (*schedule_print)(void); - -} schedule_api_t; - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_sysinfo_internal.h b/platform/linux-generic/include/odp_sysinfo_internal.h index 16e4ced84..0fef1aa32 100644 --- a/platform/linux-generic/include/odp_sysinfo_internal.h +++ b/platform/linux-generic/include/odp_sysinfo_internal.h @@ -27,9 +27,9 @@ static inline int _odp_dummy_cpuinfo(system_info_t *sysinfo) sysinfo->cpu_arch = ODP_CPU_ARCH_UNKNOWN; - ODP_DBG("Warning: use dummy values for freq and model string\n"); + _ODP_DBG("Warning: use dummy values for freq and model string\n"); for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) { - ODP_PRINT("WARN: cpu[%i] uses default max frequency of " + _ODP_PRINT("WARN: cpu[%i] uses default max frequency of " "%" PRIu64 " Hz from config file\n", i, cpu_hz_max); sysinfo->cpu_hz_max[i] = cpu_hz_max; strcpy(sysinfo->model_str[i], "UNKNOWN"); diff --git a/platform/linux-generic/odp_buffer.c b/platform/linux-generic/odp_buffer.c index 278aa8147..82bf61add 100644 --- a/platform/linux-generic/odp_buffer.c +++ b/platform/linux-generic/odp_buffer.c @@ -5,30 +5,28 @@ * SPDX-License-Identifier: BSD-3-Clause */ +#include <odp/api/align.h> #include <odp/api/buffer.h> +#include <odp/api/plat/buffer_inline_types.h> + #include <odp_pool_internal.h> #include <odp_buffer_internal.h> #include <odp_debug_internal.h> +#include <odp_print_internal.h> #include <string.h> #include <stdio.h> #include <inttypes.h> -uint32_t odp_buffer_size(odp_buffer_t buf) -{ - odp_buffer_hdr_t *hdr = _odp_buf_hdr(buf); - pool_t *pool = _odp_pool_entry(hdr->event_hdr.pool); - - return pool->seg_len; -} +#include <odp/visibility_begin.h> -void *odp_buffer_user_area(odp_buffer_t buf) -{ - odp_buffer_hdr_t *hdr = _odp_buf_hdr(buf); +/* Buffer header field offsets for inline functions */ +const _odp_buffer_inline_offset_t _odp_buffer_inline_offset ODP_ALIGNED_CACHE = { + .uarea_addr = offsetof(odp_buffer_hdr_t, uarea_addr) +}; - return hdr->uarea_addr; -} +#include <odp/visibility_end.h> void odp_buffer_print(odp_buffer_t buf) { @@ -39,21 +37,26 @@ void odp_buffer_print(odp_buffer_t buf) char str[max_len]; if (!odp_buffer_is_valid(buf)) { - ODP_ERR("Buffer is not valid.\n"); + _ODP_ERR("Buffer is not valid.\n"); return; } hdr = _odp_buf_hdr(buf); - len += snprintf(&str[len], n - len, "Buffer\n------\n"); - len += snprintf(&str[len], n - len, " pool index %u\n", hdr->event_hdr.index.pool); - len += snprintf(&str[len], n - len, " buffer index %u\n", hdr->event_hdr.index.event); - len += snprintf(&str[len], n - len, " addr %p\n", - (void *)hdr->event_hdr.base_data); - len += snprintf(&str[len], n - len, " size %u\n", odp_buffer_size(buf)); + len += _odp_snprint(&str[len], n - len, "Buffer info\n"); + len += _odp_snprint(&str[len], n - len, "-----------\n"); + len += _odp_snprint(&str[len], n - len, " handle 0x%" PRIx64 "\n", + odp_buffer_to_u64(buf)); + len += _odp_snprint(&str[len], n - len, " pool index %u\n", hdr->event_hdr.index.pool); + len += _odp_snprint(&str[len], n - len, " buffer index %u\n", + hdr->event_hdr.index.event); + len += _odp_snprint(&str[len], n - len, " addr %p\n", + (void *)hdr->event_hdr.base_data); + len += _odp_snprint(&str[len], n - len, " size %u\n", odp_buffer_size(buf)); + len += _odp_snprint(&str[len], n - len, " user area %p\n", hdr->uarea_addr); str[len] = 0; - ODP_PRINT("\n%s\n", str); + _ODP_PRINT("%s\n", str); } uint64_t odp_buffer_to_u64(odp_buffer_t hdl) diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index ed0f6723d..f1291ddcd 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -124,7 +124,7 @@ int _odp_classification_init_global(void) int _odp_classification_term_global(void) { if (_odp_cls_global && odp_shm_free(_odp_cls_global->shm)) { - ODP_ERR("shm free failed\n"); + _ODP_ERR("shm free failed\n"); return -1; } @@ -258,19 +258,19 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param_ odp_pool_info_t pool_info; if (pool == ODP_POOL_INVALID || odp_pool_info(pool, &pool_info)) { - ODP_ERR("invalid packet vector pool\n"); + _ODP_ERR("invalid packet vector pool\n"); return ODP_COS_INVALID; } if (pool_info.params.type != ODP_POOL_VECTOR) { - ODP_ERR("wrong pool type\n"); + _ODP_ERR("wrong pool type\n"); return ODP_COS_INVALID; } if (param.vector.max_size == 0) { - ODP_ERR("vector.max_size is zero\n"); + _ODP_ERR("vector.max_size is zero\n"); return ODP_COS_INVALID; } if (param.vector.max_size > pool_info.params.vector.max_size) { - ODP_ERR("vector.max_size larger than pool max vector size\n"); + _ODP_ERR("vector.max_size larger than pool max vector size\n"); return ODP_COS_INVALID; } } @@ -347,7 +347,7 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param_ UNLOCK(&cos->lock); } - ODP_ERR("CLS_COS_MAX_ENTRY reached\n"); + _ODP_ERR("CLS_COS_MAX_ENTRY reached\n"); return ODP_COS_INVALID; } @@ -363,7 +363,6 @@ odp_pmr_t alloc_pmr(pmr_t **pmr) LOCK(&pmr_tbl->pmr[i].lock); if (0 == pmr_tbl->pmr[i].valid) { pmr_tbl->pmr[i].valid = 1; - odp_atomic_init_u32(&pmr_tbl->pmr[i].count, 0); pmr_tbl->pmr[i].num_pmr = 0; *pmr = &pmr_tbl->pmr[i]; /* return as locked */ @@ -371,7 +370,7 @@ odp_pmr_t alloc_pmr(pmr_t **pmr) } UNLOCK(&pmr_tbl->pmr[i].lock); } - ODP_ERR("CLS_PMR_MAX_ENTRY reached\n"); + _ODP_ERR("CLS_PMR_MAX_ENTRY reached\n"); return ODP_PMR_INVALID; } @@ -405,7 +404,7 @@ int odp_cos_destroy(odp_cos_t cos_id) cos_t *cos = get_cos_entry(cos_id); if (NULL == cos) { - ODP_ERR("Invalid odp_cos_t handle\n"); + _ODP_ERR("Invalid odp_cos_t handle\n"); return -1; } @@ -421,12 +420,12 @@ int odp_cos_queue_set(odp_cos_t cos_id, odp_queue_t queue_id) cos_t *cos = get_cos_entry(cos_id); if (cos == NULL) { - ODP_ERR("Invalid odp_cos_t handle\n"); + _ODP_ERR("Invalid odp_cos_t handle\n"); return -1; } if (cos->num_queue != 1) { - ODP_ERR("Hashing enabled, cannot set queue\n"); + _ODP_ERR("Hashing enabled, cannot set queue\n"); return -1; } @@ -441,7 +440,7 @@ odp_queue_t odp_cos_queue(odp_cos_t cos_id) cos_t *cos = get_cos_entry(cos_id); if (!cos) { - ODP_ERR("Invalid odp_cos_t handle\n"); + _ODP_ERR("Invalid odp_cos_t handle\n"); return ODP_QUEUE_INVALID; } @@ -453,7 +452,7 @@ uint32_t odp_cls_cos_num_queue(odp_cos_t cos_id) cos_t *cos = get_cos_entry(cos_id); if (!cos) { - ODP_ERR("Invalid odp_cos_t handle\n"); + _ODP_ERR("Invalid odp_cos_t handle\n"); return 0; } @@ -470,7 +469,7 @@ uint32_t odp_cls_cos_queues(odp_cos_t cos_id, odp_queue_t queue[], cos = get_cos_entry(cos_id); if (!cos) { - ODP_ERR("Invalid odp_cos_t handle\n"); + _ODP_ERR("Invalid odp_cos_t handle\n"); return 0; } @@ -499,7 +498,7 @@ int odp_cos_drop_set(odp_cos_t cos_id, odp_cls_drop_t drop_policy) cos_t *cos = get_cos_entry(cos_id); if (!cos) { - ODP_ERR("Invalid odp_cos_t handle\n"); + _ODP_ERR("Invalid odp_cos_t handle\n"); return -1; } @@ -513,7 +512,7 @@ odp_cls_drop_t odp_cos_drop(odp_cos_t cos_id) cos_t *cos = get_cos_entry(cos_id); if (!cos) { - ODP_ERR("Invalid odp_cos_t handle\n"); + _ODP_ERR("Invalid odp_cos_t handle\n"); return -1; } @@ -527,12 +526,12 @@ int odp_pktio_default_cos_set(odp_pktio_t pktio_in, odp_cos_t default_cos) entry = get_pktio_entry(pktio_in); if (entry == NULL) { - ODP_ERR("Invalid odp_pktio_t handle\n"); + _ODP_ERR("Invalid odp_pktio_t handle\n"); return -1; } cos = get_cos_entry(default_cos); if (cos == NULL) { - ODP_ERR("Invalid odp_cos_t handle\n"); + _ODP_ERR("Invalid odp_cos_t handle\n"); return -1; } @@ -547,13 +546,13 @@ int odp_pktio_error_cos_set(odp_pktio_t pktio_in, odp_cos_t error_cos) entry = get_pktio_entry(pktio_in); if (entry == NULL) { - ODP_ERR("Invalid odp_pktio_t handle\n"); + _ODP_ERR("Invalid odp_pktio_t handle\n"); return -1; } cos = get_cos_entry(error_cos); if (cos == NULL) { - ODP_ERR("Invalid odp_cos_t handle\n"); + _ODP_ERR("Invalid odp_cos_t handle\n"); return -1; } @@ -575,7 +574,7 @@ int odp_pktio_headroom_set(odp_pktio_t pktio_in, uint32_t headroom) pktio_entry_t *entry = get_pktio_entry(pktio_in); if (entry == NULL) { - ODP_ERR("Invalid odp_pktio_t handle\n"); + _ODP_ERR("Invalid odp_pktio_t handle\n"); return -1; } entry->cls.headroom = headroom; @@ -593,7 +592,7 @@ int odp_cos_with_l2_priority(odp_pktio_t pktio_in, pktio_entry_t *entry = get_pktio_entry(pktio_in); if (entry == NULL) { - ODP_ERR("Invalid odp_pktio_t handle\n"); + _ODP_ERR("Invalid odp_pktio_t handle\n"); return -1; } l2_cos = &entry->cls.l2_cos_table; @@ -620,7 +619,7 @@ int ODP_DEPRECATE(odp_cos_with_l3_qos)(odp_pktio_t pktio_in, uint32_t num_qos, u cos_t *cos; if (entry == NULL) { - ODP_ERR("Invalid odp_pktio_t handle\n"); + _ODP_ERR("Invalid odp_pktio_t handle\n"); return -1; } @@ -649,7 +648,7 @@ static int pmr_create_term(pmr_term_value_t *value, odp_cls_pmr_term_t term = param->term; if (param->range_term) { - ODP_ERR("PMR value range not supported\n"); + _ODP_ERR("PMR value range not supported\n"); return -1; } @@ -713,13 +712,13 @@ static int pmr_create_term(pmr_term_value_t *value, break; default: - ODP_ERR("Bad PMR term\n"); + _ODP_ERR("Bad PMR term\n"); return -1; } if ((!custom && param->val_sz != size) || (custom && param->val_sz > size)) { - ODP_ERR("Bad PMR value size: %u\n", param->val_sz); + _ODP_ERR("Bad PMR value size: %u\n", param->val_sz); return -1; } @@ -776,12 +775,12 @@ static odp_pmr_t cls_pmr_create(const odp_pmr_param_t *terms, int num_terms, uin cos_t *cos_dst = get_cos_entry(dst_cos); if (NULL == cos_src || NULL == cos_dst) { - ODP_ERR("Invalid odp_cos_t handle\n"); + _ODP_ERR("Invalid odp_cos_t handle\n"); return ODP_PMR_INVALID; } if (num_terms > CLS_PMRTERM_MAX) { - ODP_ERR("no of terms greater than supported CLS_PMRTERM_MAX\n"); + _ODP_ERR("no of terms greater than supported CLS_PMRTERM_MAX\n"); return ODP_PMR_INVALID; } @@ -823,12 +822,12 @@ odp_pmr_t odp_cls_pmr_create_opt(const odp_pmr_create_opt_t *opt, odp_cos_t src_cos, odp_cos_t dst_cos) { if (opt == NULL) { - ODP_ERR("Bad parameter\n"); + _ODP_ERR("Bad parameter\n"); return ODP_PMR_INVALID; } if (opt->mark > MAX_MARK) { - ODP_ERR("Too large mark value: %" PRIu64 "\n", opt->mark); + _ODP_ERR("Too large mark value: %" PRIu64 "\n", opt->mark); return ODP_PMR_INVALID; } @@ -841,7 +840,7 @@ int odp_cls_cos_pool_set(odp_cos_t cos_id, odp_pool_t pool) cos = get_cos_entry(cos_id); if (cos == NULL) { - ODP_ERR("Invalid odp_cos_t handle\n"); + _ODP_ERR("Invalid odp_cos_t handle\n"); return -1; } @@ -856,7 +855,7 @@ odp_pool_t odp_cls_cos_pool(odp_cos_t cos_id) cos = get_cos_entry(cos_id); if (cos == NULL) { - ODP_ERR("Invalid odp_cos_t handle\n"); + _ODP_ERR("Invalid odp_cos_t handle\n"); return ODP_POOL_INVALID; } @@ -1199,7 +1198,7 @@ static inline int verify_pmr_custom_frame(const uint8_t *pkt_addr, uint32_t offset = term_value->offset; uint32_t val_sz = term_value->val_sz; - ODP_ASSERT(val_sz <= MAX_PMR_TERM_SIZE); + _ODP_ASSERT(val_sz <= MAX_PMR_TERM_SIZE); if (packet_len(pkt_hdr) <= offset + val_sz) return 0; @@ -1226,7 +1225,7 @@ static inline int verify_pmr_custom_l3(const uint8_t *pkt_addr, uint32_t offset = l3_offset + term_value->offset; uint32_t val_sz = term_value->val_sz; - ODP_ASSERT(val_sz <= MAX_PMR_TERM_SIZE); + _ODP_ASSERT(val_sz <= MAX_PMR_TERM_SIZE); if (pkt_hdr->p.input_flags.l2 == 0 || l3_offset == ODP_PACKET_OFFSET_INVALID) @@ -1446,7 +1445,6 @@ static int verify_pmr(pmr_t *pmr, const uint8_t *pkt_addr, if (pmr_failure) return 0; } - odp_atomic_inc_u32(&pmr->count); return 1; } @@ -1555,56 +1553,51 @@ static inline void pmr_debug_print(pmr_t *pmr, cos_t *cos) /* * Match a PMR chain with a Packet and return matching CoS - * This function gets called recursively to check the chained PMR Term value - * with the packet. + * This function performs a depth-first search in the CoS tree. */ -static cos_t *match_pmr_cos(cos_t *cos, const uint8_t *pkt_addr, pmr_t *pmr, - odp_packet_hdr_t *hdr) +static cos_t *match_pmr_cos(cos_t *cos, const uint8_t *pkt_addr, odp_packet_hdr_t *hdr) { - uint32_t i, num_rule; + pmr_t *pmr_match = NULL; - if (cos == NULL || pmr == NULL) - return NULL; + while (1) { + uint32_t i, num_rule = odp_atomic_load_u32(&cos->num_rule); - if (!cos->valid) - return NULL; + for (i = 0; i < num_rule; i++) { + pmr_t *pmr = cos->pmr[i]; + struct cos_s *linked_cos = cos->linked_cos[i]; - if (verify_pmr(pmr, pkt_addr, hdr)) { - /* PMR matched */ - pmr_debug_print(pmr, cos); + if (odp_unlikely(!linked_cos->valid)) + continue; - if (cos->stats_enable) - odp_atomic_inc_u64(&cos->stats.packets); + if (verify_pmr(pmr, pkt_addr, hdr)) { + /* PMR matched */ - hdr->p.input_flags.cls_mark = 0; - if (pmr->mark) { - hdr->p.input_flags.cls_mark = 1; - hdr->cls_mark = pmr->mark; - } + pmr_match = pmr; + cos = linked_cos; - /* This gets called recursively. First matching leaf or branch - * is returned. */ - num_rule = odp_atomic_load_u32(&cos->num_rule); + pmr_debug_print(pmr, cos); - /* No more rules. This is the best match. */ - if (num_rule == 0) - return cos; - - for (i = 0; i < num_rule; i++) { - cos_t *retcos = match_pmr_cos(cos->linked_cos[i], - pkt_addr, cos->pmr[i], - hdr); + if (cos->stats_enable) + odp_atomic_inc_u64(&cos->stats.packets); - /* Found a matching leaf */ - if (retcos) - return retcos; + break; + } } - /* Current CoS was the best match */ - return cos; + /* If no PMR matched, the current CoS is the best match. */ + if (i == num_rule) + break; } - return NULL; + if (pmr_match) { + hdr->p.input_flags.cls_mark = 0; + if (pmr_match->mark) { + hdr->p.input_flags.cls_mark = 1; + hdr->cls_mark = pmr_match->mark; + } + } + + return cos; } int _odp_pktio_classifier_init(pktio_entry_t *entry) @@ -1642,10 +1635,8 @@ static inline cos_t *cls_select_cos(pktio_entry_t *entry, const uint8_t *pkt_addr, odp_packet_hdr_t *pkt_hdr) { - pmr_t *pmr; cos_t *cos; cos_t *default_cos; - uint32_t i; classifier_t *cls; cls = &entry->cls; @@ -1658,11 +1649,9 @@ static inline cos_t *cls_select_cos(pktio_entry_t *entry, } /* Calls all the PMRs attached at the PKTIO level*/ - for (i = 0; i < odp_atomic_load_u32(&default_cos->num_rule); i++) { - pmr = default_cos->pmr[i]; - cos = default_cos->linked_cos[i]; - cos = match_pmr_cos(cos, pkt_addr, pmr, pkt_hdr); - if (cos) + if (default_cos && default_cos->valid) { + cos = match_pmr_cos(default_cos, pkt_addr, pkt_hdr); + if (cos && cos != default_cos) return cos; } @@ -1907,12 +1896,12 @@ int odp_cls_cos_stats(odp_cos_t hdl, odp_cls_cos_stats_t *stats) cos_t *cos = get_cos_entry(hdl); if (odp_unlikely(cos == NULL)) { - ODP_ERR("Invalid odp_cos_t handle\n"); + _ODP_ERR("Invalid odp_cos_t handle\n"); return -1; } if (odp_unlikely(stats == NULL)) { - ODP_ERR("Output structure NULL\n"); + _ODP_ERR("Output structure NULL\n"); return -1; } @@ -1930,18 +1919,18 @@ int odp_cls_queue_stats(odp_cos_t hdl, odp_queue_t queue, int queue_idx; if (odp_unlikely(cos == NULL)) { - ODP_ERR("Invalid odp_cos_t handle\n"); + _ODP_ERR("Invalid odp_cos_t handle\n"); return -1; } if (odp_unlikely(stats == NULL)) { - ODP_ERR("Output structure NULL\n"); + _ODP_ERR("Output structure NULL\n"); return -1; } queue_idx = _odp_cos_queue_idx(cos, queue); if (odp_unlikely(queue_idx < 0)) { - ODP_ERR("Invalid odp_queue_t handle\n"); + _ODP_ERR("Invalid odp_queue_t handle\n"); return -1; } @@ -1956,10 +1945,9 @@ static void print_cos_ident(cos_t *cos) { if (strlen(cos->name)) - ODP_PRINT("%s", cos->name); + _ODP_PRINT("%s", cos->name); - ODP_PRINT("(%" PRIu64 ")\n", - odp_cos_to_u64(_odp_cos_from_ndx(cos->index))); + _ODP_PRINT("(%" PRIu64 ")\n", odp_cos_to_u64(_odp_cos_from_ndx(cos->index))); } static @@ -1968,9 +1956,9 @@ void print_queue_ident(odp_queue_t q) odp_queue_info_t info; if (!odp_queue_info(q, &info) && strlen(info.name)) - ODP_PRINT(" %s\n", info.name); + _ODP_PRINT(" %s\n", info.name); else - ODP_PRINT(" %" PRIx64 "\n", odp_queue_to_u64(q)); + _ODP_PRINT(" %" PRIx64 "\n", odp_queue_to_u64(q)); } static @@ -1979,7 +1967,7 @@ void print_hex(const void *vp, int len) const uint8_t *p = vp; for (int i = 0; i < len; i++) - ODP_PRINT("%02x", *p++); + _ODP_PRINT("%02x", *p++); } static @@ -1989,9 +1977,9 @@ void cls_print_cos(cos_t *cos) uint32_t num_rule = odp_atomic_load_u32(&cos->num_rule); bool first = true; - ODP_PRINT("cos: "); + _ODP_PRINT("cos: "); print_cos_ident(cos); - ODP_PRINT(" queues:\n"); + _ODP_PRINT(" queues:\n"); if (!cos->queue_group) { print_queue_ident(cos->queue); @@ -2008,30 +1996,30 @@ void cls_print_cos(cos_t *cos) pmr_term_value_t *v = &pmr->pmr_term_value[k]; if (first) - ODP_PRINT(" rules: "); + _ODP_PRINT(" rules: "); else - ODP_PRINT(" "); + _ODP_PRINT(" "); first = false; - ODP_PRINT("%s: ", format_pmr_name(v->term)); + _ODP_PRINT("%s: ", format_pmr_name(v->term)); if (v->term == ODP_PMR_CUSTOM_FRAME || v->term == ODP_PMR_CUSTOM_L3) - ODP_PRINT("offset:%" PRIu32 " ", v->offset); + _ODP_PRINT("offset:%" PRIu32 " ", v->offset); if (v->range_term) { - ODP_PRINT("<range>"); + _ODP_PRINT("<range>"); } else { print_hex(v->match.value_u8, v->val_sz); - ODP_PRINT(" "); + _ODP_PRINT(" "); print_hex(v->match.mask_u8, v->val_sz); } - ODP_PRINT(" -> "); + _ODP_PRINT(" -> "); if (pmr->mark) - ODP_PRINT("mark:%" PRIu16 " ", pmr->mark); + _ODP_PRINT("mark:%" PRIu16 " ", pmr->mark); print_cos_ident(cos->linked_cos[j]); } @@ -2041,7 +2029,7 @@ void cls_print_cos(cos_t *cos) void odp_cls_print_all(void) { - ODP_PRINT("\n" + _ODP_PRINT("\n" "Classifier info\n" "---------------\n\n"); diff --git a/platform/linux-generic/odp_comp.c b/platform/linux-generic/odp_comp.c index 7a5b0198b..13999457f 100644 --- a/platform/linux-generic/odp_comp.c +++ b/platform/linux-generic/odp_comp.c @@ -135,14 +135,14 @@ static void process_input(odp_packet_t pkt_out, do { out_data = odp_packet_offset(pkt_out, start, &out_len, &cur_seg); - ODP_DBG("out_data %p seg_data_ptr %p out_len %d seg %p\n", - (void *)out_data, odp_packet_seg_data(pkt_out, cur_seg), - out_len, (void *)cur_seg); + _ODP_DBG("out_data %p seg_data_ptr %p out_len %d seg %p\n", + (void *)out_data, odp_packet_seg_data(pkt_out, cur_seg), + out_len, (void *)cur_seg); if (0 == out_len) { /* there are no more segments */ - ODP_DBG("Ran out of space. (streamp->avail_out) %d\n", - (streamp->avail_out)); + _ODP_DBG("Ran out of space. (streamp->avail_out) %d\n", + (streamp->avail_out)); result->status = ODP_COMP_STATUS_OUT_OF_SPACE_TERM; break; } @@ -156,12 +156,9 @@ static void process_input(odp_packet_t pkt_out, streamp->next_out = out_data; streamp->avail_out = out_len; - ODP_DBG("next_in %p, avail_in %d next_out %p" - " avail_out %d, sync %d\n", - (const void *)streamp->next_in, streamp->avail_in, - (void *)streamp->next_out, - streamp->avail_out, - sync); + _ODP_DBG("next_in %p, avail_in %d next_out %p avail_out %d, sync %d\n", + (const void *)streamp->next_in, streamp->avail_in, + (void *)streamp->next_out, streamp->avail_out, sync); if (session->params.op == ODP_COMP_OP_COMPRESS) ret = mz_deflate(streamp, @@ -169,8 +166,8 @@ static void process_input(odp_packet_t pkt_out, else ret = mz_inflate(streamp, MZ_NO_FLUSH); - ODP_DBG("ret %d streamp->avail_out %d avail_in %d\n", - ret, streamp->avail_out, streamp->avail_in); + _ODP_DBG("ret %d streamp->avail_out %d avail_in %d\n", + ret, streamp->avail_out, streamp->avail_in); out_len = out_len - streamp->avail_out; written += out_len; @@ -182,7 +179,7 @@ static void process_input(odp_packet_t pkt_out, start += out_len; space_avail -= out_len; - ODP_DBG("ret %d,written %d\n", ret, out_len); + _ODP_DBG("ret %d,written %d\n", ret, out_len); if (ret == MZ_STREAM_END) { if (session->params.op == ODP_COMP_OP_COMPRESS) { @@ -196,7 +193,7 @@ static void process_input(odp_packet_t pkt_out, break; } if ((ret != MZ_BUF_ERROR) && (ret != MZ_OK)) { - ODP_DBG("deflate failed. Err %s,ret %d" + _ODP_DBG("deflate failed. Err %s,ret %d" "(streamp->avail_out) %d\n", streamp->msg, ret, (streamp->avail_out)); result->status = ODP_COMP_STATUS_FAILURE; @@ -210,7 +207,7 @@ static void process_input(odp_packet_t pkt_out, /* if write stopped as output exhausted, return OUT_OF_SPACE_ERR */ - ODP_DBG("Ran out of space. (out avail) %d," + _ODP_DBG("Ran out of space. (out avail) %d," "to process %d\n", streamp->avail_out, streamp->avail_in); result->status = ODP_COMP_STATUS_OUT_OF_SPACE_TERM; @@ -239,10 +236,10 @@ static int deflate_comp(odp_packet_t pkt_in, odp_packet_seg_t in_seg = ODP_PACKET_SEG_INVALID; odp_comp_packet_result_t *result = get_op_result_from_packet(pkt_out); - ODP_ASSERT(session != NULL); - ODP_ASSERT(params != NULL); - ODP_ASSERT(pkt_in != ODP_PACKET_INVALID); - ODP_ASSERT(pkt_out != ODP_PACKET_INVALID); + _ODP_ASSERT(session != NULL); + _ODP_ASSERT(params != NULL); + _ODP_ASSERT(pkt_in != ODP_PACKET_INVALID); + _ODP_ASSERT(pkt_out != ODP_PACKET_INVALID); streamp = &session->comp.stream; @@ -260,8 +257,8 @@ static int deflate_comp(odp_packet_t pkt_in, read, &in_len, &in_seg); - ODP_DBG("data %p in_len %d seg %p len %d\n", - (void *)data, in_len, (void *)in_seg, len); + _ODP_DBG("data %p in_len %d seg %p len %d\n", + (void *)data, in_len, (void *)in_seg, len); if (in_len > len) in_len = len; @@ -272,7 +269,7 @@ static int deflate_comp(odp_packet_t pkt_in, streamp->avail_in = in_len; if (consumed >= len) { - ODP_DBG("This is last chunk\n"); + _ODP_DBG("This is last chunk\n"); sync = true; } @@ -284,9 +281,7 @@ static int deflate_comp(odp_packet_t pkt_in, read += in_len; } - ODP_DBG("Read %d Written %d\n", - read, - result->output_data_range.length); + _ODP_DBG("Read %d Written %d\n", read, result->output_data_range.length); return 0; } @@ -316,7 +311,7 @@ static int deflate_init(odp_comp_generic_session_t *session) odp_comp_huffman_code_t cc; /* optional check as such may not required */ - ODP_ASSERT(strcmp(mz_version(), MZ_VERSION) == 0); + _ODP_ASSERT(strcmp(mz_version(), MZ_VERSION) == 0); memset(&session->comp.stream, 0, sizeof(mz_stream)); @@ -358,20 +353,17 @@ static int deflate_init(odp_comp_generic_session_t *session) default: return -1; } - ODP_DBG(" level %d strategy %d window %d\n", - level, strategy, window_bits); + _ODP_DBG(" level %d strategy %d window %d\n", level, strategy, window_bits); if (ODP_COMP_OP_COMPRESS == session->params.op) { if (mz_deflateInit2(streamp, level, MZ_DEFLATED, window_bits, MEM_LEVEL, strategy) != MZ_OK) { - ODP_DBG("Err in Deflate Initialization %s\n", - streamp->msg); + _ODP_DBG("Err in Deflate Initialization %s\n", streamp->msg); return -1; } } else { if (mz_inflateInit2(streamp, window_bits) != MZ_OK) { - ODP_DBG("Err in Inflate Initialization %s\n", - streamp->msg); + _ODP_DBG("Err in Inflate Initialization %s\n", streamp->msg); return -1; } } @@ -390,14 +382,13 @@ static int term_def(odp_comp_generic_session_t *session) rc = mz_deflateEnd(streamp); if (rc != MZ_OK) { - ODP_ERR("deflateEnd failed. Err %s,rc %d\n", - streamp->msg, rc); + _ODP_ERR("deflateEnd failed. Err %s,rc %d\n", streamp->msg, rc); /* we choose to just return 0 with error info */ } } else { rc = mz_inflateEnd(streamp); if (rc != MZ_OK) { - ODP_ERR("inflateEnd failed. Err %s\n", streamp->msg); + _ODP_ERR("inflateEnd failed. Err %s\n", streamp->msg); /* we choose to just return 0 with error info */ } } @@ -459,7 +450,7 @@ int odp_comp_session_destroy(odp_comp_session_t session) break; } if (rc < 0) { - ODP_ERR("Compression Unit could not be terminated\n"); + _ODP_ERR("Compression Unit could not be terminated\n"); return -1; } @@ -533,12 +524,12 @@ static int _odp_comp_single(odp_packet_t pkt_in, odp_packet_t pkt_out, int rc; session = to_gen_session(param->session); - ODP_ASSERT(session); - ODP_ASSERT(pkt_in != ODP_PACKET_INVALID); - ODP_ASSERT(pkt_out != ODP_PACKET_INVALID); + _ODP_ASSERT(session); + _ODP_ASSERT(pkt_in != ODP_PACKET_INVALID); + _ODP_ASSERT(pkt_out != ODP_PACKET_INVALID); result = get_op_result_from_packet(pkt_out); - ODP_ASSERT(result); + _ODP_ASSERT(result); result->pkt_in = pkt_in; result->output_data_range.offset = param->out_data_range.offset; @@ -598,11 +589,11 @@ int odp_comp_result(odp_comp_packet_result_t *result, { odp_comp_packet_result_t *op_result; - ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) + _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == ODP_EVENT_PACKET_COMP); op_result = get_op_result_from_packet(packet); - ODP_DBG("Copy operational result back\n"); + _ODP_DBG("Copy operational result back\n"); memcpy(result, op_result, sizeof(*result)); return 0; } @@ -614,7 +605,7 @@ int _odp_comp_init_global(void) int idx; if (odp_global_ro.disable.compress) { - ODP_PRINT("\nODP compress is DISABLED\n"); + _ODP_PRINT("\nODP compress is DISABLED\n"); return 0; } @@ -655,13 +646,13 @@ int _odp_comp_term_global(void) count++; if (count != MAX_SESSIONS) { - ODP_ERR("comp sessions still active\n"); + _ODP_ERR("comp sessions still active\n"); rc = -1; } ret = odp_shm_free(global->global_shm); if (ret < 0) { - ODP_ERR("shm free failed for comp_pool\n"); + _ODP_ERR("shm free failed for comp_pool\n"); rc = -1; } @@ -671,8 +662,8 @@ int _odp_comp_term_global(void) odp_packet_t odp_comp_packet_from_event(odp_event_t ev) { /* This check not mandated by the API specification */ - ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); - ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_COMP); + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); + _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_COMP); return odp_packet_from_event(ev); } diff --git a/platform/linux-generic/odp_cpumask.c b/platform/linux-generic/odp_cpumask.c index be792e82a..d0a9953f7 100644 --- a/platform/linux-generic/odp_cpumask.c +++ b/platform/linux-generic/odp_cpumask.c @@ -238,7 +238,7 @@ static int get_available_cpus(void) ret = sched_getaffinity(0, sizeof(cpuset), &cpuset); if (ret < 0) { - ODP_ERR("Failed to get cpu affinity"); + _ODP_ERR("Failed to get cpu affinity"); return -1; } diff --git a/platform/linux-generic/odp_cpumask_task.c b/platform/linux-generic/odp_cpumask_task.c index 9d0b03fd9..0807e231e 100644 --- a/platform/linux-generic/odp_cpumask_task.c +++ b/platform/linux-generic/odp_cpumask_task.c @@ -7,11 +7,13 @@ #include <odp_posix_extensions.h> -#include <sched.h> -#include <pthread.h> - #include <odp/api/cpumask.h> + #include <odp_debug_internal.h> +#include <odp_global_data.h> + +#include <pthread.h> +#include <sched.h> int odp_cpumask_default_worker(odp_cpumask_t *mask, int max_num) { diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index 981c9239c..b9d319861 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -116,7 +116,7 @@ void free_session(odp_crypto_generic_session_t *session) int odp_crypto_capability(odp_crypto_capability_t *capa) { if (odp_global_ro.disable.crypto) { - ODP_ERR("Crypto is disabled\n"); + _ODP_ERR("Crypto is disabled\n"); return -1; } @@ -198,7 +198,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, odp_crypto_generic_session_t *session; if (odp_global_ro.disable.crypto) { - ODP_ERR("Crypto is disabled\n"); + _ODP_ERR("Crypto is disabled\n"); /* Dummy output to avoid compiler warning about uninitialized * variables */ *status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM; @@ -333,7 +333,7 @@ _odp_crypto_init_global(void) int idx; if (odp_global_ro.disable.crypto) { - ODP_PRINT("\nODP crypto is DISABLED\n"); + _ODP_PRINT("\nODP crypto is DISABLED\n"); return 0; } @@ -345,7 +345,7 @@ _odp_crypto_init_global(void) ODP_CACHE_LINE_SIZE, 0); if (ODP_SHM_INVALID == shm) { - ODP_ERR("unable to allocate crypto pool\n"); + _ODP_ERR("unable to allocate crypto pool\n"); return -1; } @@ -377,13 +377,13 @@ int _odp_crypto_term_global(void) for (session = global->free; session != NULL; session = session->next) count++; if (count != MAX_SESSIONS) { - ODP_ERR("crypto sessions still active\n"); + _ODP_ERR("crypto sessions still active\n"); rc = -1; } ret = odp_shm_free(odp_shm_lookup("_odp_crypto_null_global")); if (ret < 0) { - ODP_ERR("shm free failed for _odp_crypto_pool_null\n"); + _ODP_ERR("shm free failed for _odp_crypto_pool_null\n"); rc = -1; } @@ -405,7 +405,7 @@ odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev) { /* This check not mandated by the API specification */ if (odp_event_type(ev) != ODP_EVENT_CRYPTO_COMPL) - ODP_ABORT("Event not a crypto completion"); + _ODP_ABORT("Event not a crypto completion"); return (odp_crypto_compl_t)ev; } @@ -422,7 +422,7 @@ odp_crypto_compl_result(odp_crypto_compl_t completion_event, (void)result; /* We won't get such events anyway, so there can be no result */ - ODP_ASSERT(0); + _ODP_ASSERT(0); } void @@ -452,8 +452,8 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) odp_packet_t odp_crypto_packet_from_event(odp_event_t ev) { /* This check not mandated by the API specification */ - ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); - ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); + _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); return odp_packet_from_event(ev); } @@ -476,7 +476,7 @@ int odp_crypto_result(odp_crypto_packet_result_t *result, { odp_crypto_packet_result_t *op_result; - ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == + _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == ODP_EVENT_PACKET_CRYPTO); op_result = get_op_result_from_packet(packet); @@ -494,13 +494,13 @@ static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src) md_copy = _odp_packet_copy_md_possible(odp_packet_pool(dst), odp_packet_pool(src)); if (odp_unlikely(md_copy < 0)) { - ODP_ERR("Unable to copy packet metadata\n"); + _ODP_ERR("Unable to copy packet metadata\n"); return -1; } rc = odp_packet_copy_from_pkt(dst, 0, src, 0, odp_packet_len(src)); if (odp_unlikely(rc < 0)) { - ODP_ERR("Unable to copy packet data\n"); + _ODP_ERR("Unable to copy packet data\n"); return -1; } @@ -520,7 +520,7 @@ static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *sessio if (pkt_out == ODP_PACKET_INVALID) { odp_pool_t pool = session->p.output_pool; - ODP_ASSERT(pool != ODP_POOL_INVALID); + _ODP_ASSERT(pool != ODP_POOL_INVALID); if (pool == odp_packet_pool(pkt_in)) { pkt_out = pkt_in; } else { @@ -578,7 +578,7 @@ int odp_crypto_op(const odp_packet_t pkt_in[], for (i = 0; i < num_pkt; i++) { session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session; - ODP_ASSERT(ODP_CRYPTO_SYNC == session->p.op_mode); + _ODP_ASSERT(ODP_CRYPTO_SYNC == session->p.op_mode); rc = crypto_int(pkt_in[i], &pkt_out[i], ¶m[i]); if (rc < 0) @@ -600,8 +600,8 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[], for (i = 0; i < num_pkt; i++) { session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session; - ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); - ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); + _ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); + _ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); pkt = pkt_out[i]; rc = crypto_int(pkt_in[i], &pkt, ¶m[i]); diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 9b5ea4612..0f637850f 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -384,7 +384,7 @@ void packet_hmac(odp_packet_t pkt, uint32_t offset = param->auth_range.offset; uint32_t len = param->auth_range.length; - ODP_ASSERT(offset + len <= odp_packet_len(pkt)); + _ODP_ASSERT(offset + len <= odp_packet_len(pkt)); /* Reinitialize HMAC calculation without resetting the key */ HMAC_Init_ex(ctx, NULL, 0, NULL, NULL); @@ -434,9 +434,9 @@ void packet_aes_xcbc_mac(odp_packet_t pkt, void *mapaddr; uint8_t *data = NULL; - ODP_ASSERT(offset + len <= odp_packet_len(pkt)); - ODP_ASSERT(session != NULL); - ODP_ASSERT(sizeof(session->auth.key) >= 3 * AES_KEY_LENGTH); + _ODP_ASSERT(offset + len <= odp_packet_len(pkt)); + _ODP_ASSERT(session != NULL); + _ODP_ASSERT(sizeof(session->auth.key) >= 3 * AES_KEY_LENGTH); ctx = EVP_CIPHER_CTX_new(); EVP_EncryptInit_ex(ctx, session->auth.evp_cipher, @@ -639,7 +639,7 @@ void packet_cmac(odp_packet_t pkt, uint32_t len = param->auth_range.length; size_t outlen; - ODP_ASSERT(offset + len <= odp_packet_len(pkt)); + _ODP_ASSERT(offset + len <= odp_packet_len(pkt)); /* Reinitialize CMAC calculation without resetting the key */ CMAC_Init(ctx, NULL, 0, NULL, NULL); @@ -724,10 +724,10 @@ int packet_cmac_eia2(odp_packet_t pkt, return ODP_CRYPTO_ALG_ERR_IV_INVALID; #else iv_ptr = param->auth_iv_ptr; - ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); + _ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); #endif - ODP_ASSERT(offset + len <= odp_packet_len(pkt)); + _ODP_ASSERT(offset + len <= odp_packet_len(pkt)); /* Reinitialize CMAC calculation without resetting the key */ CMAC_Init(ctx, NULL, 0, NULL, NULL); @@ -814,7 +814,7 @@ void packet_digest(odp_packet_t pkt, uint32_t offset = param->auth_range.offset; uint32_t len = param->auth_range.length; - ODP_ASSERT(offset + len <= odp_packet_len(pkt)); + _ODP_ASSERT(offset + len <= odp_packet_len(pkt)); EVP_DigestInit_ex(ctx, session->auth.evp_md, @@ -892,7 +892,7 @@ int internal_aad(EVP_CIPHER_CTX *ctx, int dummy_len; int ret; - ODP_ASSERT(offset + len <= odp_packet_len(pkt)); + _ODP_ASSERT(offset + len <= odp_packet_len(pkt)); while (len > 0) { uint32_t seglen = 0; /* GCC */ @@ -935,7 +935,7 @@ static inline int internal_crypt(EVP_CIPHER_CTX *ctx, int out_len; int rc; - ODP_ASSERT(in_pos + in_len <= odp_packet_len(pkt)); + _ODP_ASSERT(in_pos + in_len <= odp_packet_len(pkt)); /* * In the following loop we process one packet segment per iteration. @@ -1002,7 +1002,7 @@ static inline int internal_crypt(EVP_CIPHER_CTX *ctx, continue; } } - ODP_ASSERT(buffered == 0); + _ODP_ASSERT(buffered == 0); if (in_len > 0) { /* @@ -1014,7 +1014,7 @@ static inline int internal_crypt(EVP_CIPHER_CTX *ctx, rc = EVP_update(ctx, in_addr, &out_len, in_addr, len); if (odp_unlikely(rc != 1)) goto err; - ODP_ASSERT(_ODP_CHECK_IS_POWER2(block_len)); + _ODP_ASSERT(_ODP_CHECK_IS_POWER2(block_len)); buffered = len & (block_len - 1); if (odp_unlikely(out_len + buffered != len)) goto err; @@ -1034,7 +1034,7 @@ static inline int internal_crypt(EVP_CIPHER_CTX *ctx, return 0; return rc; err: - ODP_ERR("internal error\n"); + _ODP_ERR("internal error\n"); (void)EVP_final(ctx, block, &out_len); return 0; } @@ -1085,7 +1085,7 @@ odp_crypto_alg_err_t cipher_encrypt(odp_packet_t pkt, return ODP_CRYPTO_ALG_ERR_IV_INVALID; #else iv_ptr = param->cipher_iv_ptr; - ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); + _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); #endif EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); @@ -1124,7 +1124,7 @@ odp_crypto_alg_err_t cipher_decrypt(odp_packet_t pkt, return ODP_CRYPTO_ALG_ERR_IV_INVALID; #else iv_ptr = param->cipher_iv_ptr; - ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); + _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); #endif EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); @@ -1192,7 +1192,7 @@ odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt, return ODP_CRYPTO_ALG_ERR_IV_INVALID; #else iv_ptr = param->cipher_iv_ptr; - ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); + _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); #endif EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); @@ -1236,7 +1236,7 @@ odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt, return ODP_CRYPTO_ALG_ERR_IV_INVALID; #else iv_ptr = param->cipher_iv_ptr; - ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); + _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); #endif EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); @@ -1317,7 +1317,7 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt, return ODP_CRYPTO_ALG_ERR_IV_INVALID; #else iv_ptr = param->cipher_iv_ptr; - ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); + _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); #endif EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); @@ -1372,7 +1372,7 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt, return ODP_CRYPTO_ALG_ERR_IV_INVALID; #else iv_ptr = param->cipher_iv_ptr; - ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); + _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); #endif EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); @@ -1453,7 +1453,7 @@ odp_crypto_alg_err_t aes_gmac_gen(odp_packet_t pkt, return ODP_CRYPTO_ALG_ERR_IV_INVALID; #else iv_ptr = param->auth_iv_ptr; - ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); + _ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); #endif EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); @@ -1500,7 +1500,7 @@ odp_crypto_alg_err_t aes_gmac_check(odp_packet_t pkt, return ODP_CRYPTO_ALG_ERR_IV_INVALID; #else iv_ptr = param->auth_iv_ptr; - ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); + _ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); #endif EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); @@ -1585,7 +1585,7 @@ odp_crypto_alg_err_t aes_ccm_encrypt(odp_packet_t pkt, return ODP_CRYPTO_ALG_ERR_IV_INVALID; #else iv_ptr = param->cipher_iv_ptr; - ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); + _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); #endif EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, @@ -1657,7 +1657,7 @@ odp_crypto_alg_err_t aes_ccm_decrypt(odp_packet_t pkt, return ODP_CRYPTO_ALG_ERR_IV_INVALID; #else iv_ptr = param->cipher_iv_ptr; - ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); + _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); #endif odp_packet_copy_to_mem(pkt, param->hash_result_offset, @@ -1741,7 +1741,7 @@ odp_crypto_alg_err_t xts_encrypt(odp_packet_t pkt, return ODP_CRYPTO_ALG_ERR_IV_INVALID; #else iv_ptr = param->cipher_iv_ptr; - ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); + _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); #endif EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); @@ -1783,7 +1783,7 @@ odp_crypto_alg_err_t xts_decrypt(odp_packet_t pkt, return ODP_CRYPTO_ALG_ERR_IV_INVALID; #else iv_ptr = param->cipher_iv_ptr; - ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); + _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); #endif EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); @@ -1947,7 +1947,7 @@ static int process_digest_param(odp_crypto_generic_session_t *session, int odp_crypto_capability(odp_crypto_capability_t *capa) { if (odp_global_ro.disable.crypto) { - ODP_ERR("Crypto is disabled\n"); + _ODP_ERR("Crypto is disabled\n"); return -1; } @@ -2194,7 +2194,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, odp_crypto_generic_session_t *session; if (odp_global_ro.disable.crypto) { - ODP_ERR("Crypto is disabled\n"); + _ODP_ERR("Crypto is disabled\n"); /* Dummy output to avoid compiler warning about uninitialized * variables */ *status = ODP_CRYPTO_SES_ERR_ENOMEM; @@ -2223,13 +2223,13 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, session->p = *param; if (session->p.cipher_iv_len > EVP_MAX_IV_LENGTH) { - ODP_DBG("Maximum IV length exceeded\n"); + _ODP_DBG("Maximum IV length exceeded\n"); *status = ODP_CRYPTO_SES_ERR_CIPHER; goto err; } if (session->p.auth_iv_len > EVP_MAX_IV_LENGTH) { - ODP_DBG("Maximum auth IV length exceeded\n"); + _ODP_DBG("Maximum auth IV length exceeded\n"); *status = ODP_CRYPTO_SES_ERR_CIPHER; goto err; } @@ -2599,7 +2599,7 @@ int _odp_crypto_init_global(void) int nlocks = CRYPTO_num_locks(); if (odp_global_ro.disable.crypto) { - ODP_PRINT("\nODP crypto is DISABLED\n"); + _ODP_PRINT("\nODP crypto is DISABLED\n"); return 0; } @@ -2612,7 +2612,7 @@ int _odp_crypto_init_global(void) ODP_CACHE_LINE_SIZE, 0); if (ODP_SHM_INVALID == shm) { - ODP_ERR("unable to allocate crypto pool\n"); + _ODP_ERR("unable to allocate crypto pool\n"); return -1; } @@ -2654,7 +2654,7 @@ int _odp_crypto_term_global(void) for (session = global->free; session != NULL; session = session->next) count++; if (count != MAX_SESSIONS) { - ODP_ERR("crypto sessions still active\n"); + _ODP_ERR("crypto sessions still active\n"); rc = -1; } @@ -2665,7 +2665,7 @@ int _odp_crypto_term_global(void) ret = odp_shm_free(odp_shm_lookup("_odp_crypto_ssl_global")); if (ret < 0) { - ODP_ERR("shm free failed for crypto_pool\n"); + _ODP_ERR("shm free failed for crypto_pool\n"); rc = -1; } @@ -2734,7 +2734,7 @@ odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev) { /* This check not mandated by the API specification */ if (odp_event_type(ev) != ODP_EVENT_CRYPTO_COMPL) - ODP_ABORT("Event not a crypto completion"); + _ODP_ABORT("Event not a crypto completion"); return (odp_crypto_compl_t)ev; } @@ -2751,7 +2751,7 @@ odp_crypto_compl_result(odp_crypto_compl_t completion_event, (void)result; /* We won't get such events anyway, so there can be no result */ - ODP_ASSERT(0); + _ODP_ASSERT(0); } void @@ -2781,8 +2781,8 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) odp_packet_t odp_crypto_packet_from_event(odp_event_t ev) { /* This check not mandated by the API specification */ - ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); - ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); + _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); return odp_packet_from_event(ev); } @@ -2805,7 +2805,7 @@ int odp_crypto_result(odp_crypto_packet_result_t *result, { odp_crypto_packet_result_t *op_result; - ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == + _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == ODP_EVENT_PACKET_CRYPTO); op_result = get_op_result_from_packet(packet); @@ -2823,13 +2823,13 @@ static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src) md_copy = _odp_packet_copy_md_possible(odp_packet_pool(dst), odp_packet_pool(src)); if (odp_unlikely(md_copy < 0)) { - ODP_ERR("Unable to copy packet metadata\n"); + _ODP_ERR("Unable to copy packet metadata\n"); return -1; } rc = odp_packet_copy_from_pkt(dst, 0, src, 0, odp_packet_len(src)); if (odp_unlikely(rc < 0)) { - ODP_ERR("Unable to copy packet data\n"); + _ODP_ERR("Unable to copy packet data\n"); return -1; } @@ -2849,7 +2849,7 @@ static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *sessio if (pkt_out == ODP_PACKET_INVALID) { odp_pool_t pool = session->p.output_pool; - ODP_ASSERT(pool != ODP_POOL_INVALID); + _ODP_ASSERT(pool != ODP_POOL_INVALID); if (pool == odp_packet_pool(pkt_in)) { pkt_out = pkt_in; } else { @@ -2884,6 +2884,17 @@ int crypto_int(odp_packet_t pkt_in, if (odp_unlikely(out_pkt == ODP_PACKET_INVALID)) return -1; + if (ODP_DEBUG) { + if (session->p.auth_alg != ODP_AUTH_ALG_NULL && + param->hash_result_offset + session->p.auth_digest_len + > odp_packet_len(out_pkt)) { + _ODP_ERR("Invalid hash result offset\n"); + rc_cipher = ODP_CRYPTO_ALG_ERR_DATA_SIZE; + rc_auth = ODP_CRYPTO_ALG_ERR_DATA_SIZE; + goto out; + } + } + crypto_init(session); /* Invoke the functions */ @@ -2895,6 +2906,7 @@ int crypto_int(odp_packet_t pkt_in, rc_cipher = session->cipher.func(out_pkt, param, session); } +out: /* Fill in result */ packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO); op_result = get_op_result_from_packet(out_pkt); @@ -2922,7 +2934,7 @@ int odp_crypto_op(const odp_packet_t pkt_in[], for (i = 0; i < num_pkt; i++) { session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session; - ODP_ASSERT(ODP_CRYPTO_SYNC == session->p.op_mode); + _ODP_ASSERT(ODP_CRYPTO_SYNC == session->p.op_mode); rc = crypto_int(pkt_in[i], &pkt_out[i], ¶m[i]); if (rc < 0) @@ -2944,8 +2956,8 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[], for (i = 0; i < num_pkt; i++) { session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session; - ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); - ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); + _ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); + _ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); pkt = pkt_out[i]; rc = crypto_int(pkt_in[i], &pkt, ¶m[i]); diff --git a/platform/linux-generic/odp_dma.c b/platform/linux-generic/odp_dma.c index 4c129c616..cf426f104 100644 --- a/platform/linux-generic/odp_dma.c +++ b/platform/linux-generic/odp_dma.c @@ -5,6 +5,7 @@ */ #include <odp/api/dma.h> +#include <odp/api/event.h> #include <odp/api/shared_memory.h> #include <odp/api/ticketlock.h> #include <odp/api/align.h> @@ -79,7 +80,7 @@ static inline dma_session_t *dma_session_from_handle(odp_dma_t dma) int odp_dma_capability(odp_dma_capability_t *capa) { if (odp_global_ro.disable.dma) { - ODP_ERR("DMA is disabled\n"); + _ODP_ERR("DMA is disabled\n"); return -1; } @@ -131,7 +132,7 @@ static odp_stash_t create_stash(void) stash = odp_stash_create("_odp_dma_transfer_id", &stash_param); if (stash == ODP_STASH_INVALID) { - ODP_ERR("Stash create failed\n"); + _ODP_ERR("Stash create failed\n"); return ODP_STASH_INVALID; } @@ -139,7 +140,7 @@ static odp_stash_t create_stash(void) for (id = 1; id < MAX_TRANSFERS + 1; id++) { ret = odp_stash_put_u32(stash, &id, 1); if (ret != 1) { - ODP_ERR("Stash put failed: %i, %u\n", ret, id); + _ODP_ERR("Stash put failed: %i, %u\n", ret, id); break; } } @@ -147,13 +148,13 @@ static odp_stash_t create_stash(void) if (ret != 1) { for (i = 0; i < id; i++) { if (odp_stash_get_u32(stash, &tmp, 1) != 1) { - ODP_ERR("Stash get failed: %u\n", i); + _ODP_ERR("Stash get failed: %u\n", i); break; } } if (odp_stash_destroy(stash)) - ODP_ERR("Stash destroy failed\n"); + _ODP_ERR("Stash destroy failed\n"); return ODP_STASH_INVALID; } @@ -176,13 +177,13 @@ static int destroy_stash(odp_stash_t stash) if (num == 0) break; - ODP_ERR("Stash get failed: %i\n", num); + _ODP_ERR("Stash get failed: %i\n", num); ret = -1; break; } if (odp_stash_destroy(stash)) { - ODP_ERR("Stash destroy failed\n"); + _ODP_ERR("Stash destroy failed\n"); ret = -1; } @@ -196,28 +197,28 @@ odp_dma_t odp_dma_create(const char *name, const odp_dma_param_t *param) dma_session_t *session = NULL; if (odp_global_ro.disable.dma) { - ODP_ERR("DMA is disabled\n"); + _ODP_ERR("DMA is disabled\n"); return ODP_DMA_INVALID; } if ((param->direction != ODP_DMA_MAIN_TO_MAIN) || (param->type != ODP_DMA_TYPE_COPY)) { - ODP_ERR("Bad DMA parameter\n"); + _ODP_ERR("Bad DMA parameter\n"); return ODP_DMA_INVALID; } if (param->compl_mode_mask == 0) { - ODP_ERR("Empty compl mode mask\n"); + _ODP_ERR("Empty compl mode mask\n"); return ODP_DMA_INVALID; } if (odp_dma_capability(&dma_capa)) { - ODP_ERR("DMA capa failed\n"); + _ODP_ERR("DMA capa failed\n"); return ODP_DMA_INVALID; } if (param->compl_mode_mask & ~dma_capa.compl_mode_mask) { - ODP_ERR("Compl mode not supported\n"); + _ODP_ERR("Compl mode not supported\n"); return ODP_DMA_INVALID; } @@ -239,7 +240,7 @@ odp_dma_t odp_dma_create(const char *name, const odp_dma_param_t *param) } if (session == NULL) { - ODP_DBG("Out of DMA sessions\n"); + _ODP_DBG("Out of DMA sessions\n"); return ODP_DMA_INVALID; } @@ -271,7 +272,7 @@ int odp_dma_destroy(odp_dma_t dma) int ret = 0; if (dma == ODP_DMA_INVALID) { - ODP_ERR("Bad DMA handle\n"); + _ODP_ERR("Bad DMA handle\n"); return -1; } @@ -282,7 +283,7 @@ int odp_dma_destroy(odp_dma_t dma) odp_ticketlock_lock(&session->lock); if (session->active == 0) { - ODP_ERR("Session not created\n"); + _ODP_ERR("Session not created\n"); odp_ticketlock_unlock(&session->lock); return -1; } @@ -372,7 +373,7 @@ static inline int segment_pkt(segment_t seg[], int num_seg, const odp_dma_seg_t tot_len = dma_seg[i].len; if (odp_unlikely(offset + tot_len > odp_packet_len(pkt))) { - ODP_ERR("Bad packet segment len/offset (%u/%u)\n", tot_len, offset); + _ODP_ERR("Bad packet segment len/offset (%u/%u)\n", tot_len, offset); return 0; } @@ -380,7 +381,7 @@ static inline int segment_pkt(segment_t seg[], int num_seg, const odp_dma_seg_t addr = odp_packet_offset(pkt, offset, &seg_len, NULL); if (odp_unlikely(addr == NULL)) { - ODP_ERR("Bad packet offset %u\n", offset); + _ODP_ERR("Bad packet offset %u\n", offset); return 0; } @@ -396,7 +397,7 @@ static inline int segment_pkt(segment_t seg[], int num_seg, const odp_dma_seg_t num++; if (odp_unlikely(num >= MAX_SEGS)) { - ODP_ERR("Too many packet segments\n"); + _ODP_ERR("Too many packet segments\n"); return 0; } } @@ -474,29 +475,29 @@ int odp_dma_transfer(odp_dma_t dma, const odp_dma_transfer_param_t *transfer, segment_t dst[MAX_SEGS]; if (odp_unlikely(dma == ODP_DMA_INVALID)) { - ODP_ERR("Bad DMA handle\n"); + _ODP_ERR("Bad DMA handle\n"); return -1; } if (odp_unlikely(session->active == 0)) { - ODP_ERR("Session not created\n"); + _ODP_ERR("Session not created\n"); return -1; } if (odp_unlikely(transfer->num_src == 0 || transfer->num_src > MAX_SEGS)) { - ODP_ERR("Bad number of src segments\n"); + _ODP_ERR("Bad number of src segments\n"); return -1; } if (odp_unlikely(transfer->num_dst == 0 || transfer->num_dst > MAX_SEGS)) { - ODP_ERR("Bad number of dst segments\n"); + _ODP_ERR("Bad number of dst segments\n"); return -1; } tot_len = transfer_len(transfer); if (odp_unlikely(tot_len == 0)) { - ODP_ERR("Bad transfer length\n"); + _ODP_ERR("Bad transfer length\n"); return -1; } @@ -523,7 +524,7 @@ int odp_dma_transfer(odp_dma_t dma, const odp_dma_transfer_param_t *transfer, num = transfer_table(trs, src, dst, max_num, tot_len); if (odp_unlikely(num > max_num)) { - ODP_ERR("Segment table error\n"); + _ODP_ERR("Segment table error\n"); return -1; } @@ -546,7 +547,7 @@ int odp_dma_transfer_multi(odp_dma_t dma, const odp_dma_transfer_param_t *trs_pa int ret = 0; if (odp_unlikely(num < 1)) { - ODP_ERR("Bad number of transfers\n"); + _ODP_ERR("Bad number of transfers\n"); return -1; } @@ -597,7 +598,7 @@ void odp_dma_transfer_id_free(odp_dma_t dma, odp_dma_transfer_id_t transfer_id) num = odp_stash_put_u32(session->stash, &id, 1); if (odp_unlikely(num != 1)) - ODP_ERR("Stash put failed\n"); + _ODP_ERR("Stash put failed\n"); } static inline uint32_t index_from_transfer_id(odp_dma_transfer_id_t transfer_id) @@ -612,7 +613,7 @@ int odp_dma_transfer_start(odp_dma_t dma, const odp_dma_transfer_param_t *transf dma_session_t *session = dma_session_from_handle(dma); if (odp_unlikely(dma == ODP_DMA_INVALID)) { - ODP_ERR("Bad DMA handle\n"); + _ODP_ERR("Bad DMA handle\n"); return -1; } @@ -623,19 +624,19 @@ int odp_dma_transfer_start(odp_dma_t dma, const odp_dma_transfer_param_t *transf case ODP_DMA_COMPL_POLL: if (compl->transfer_id == ODP_DMA_TRANSFER_ID_INVALID || compl->transfer_id > MAX_TRANSFERS) { - ODP_ERR("Bad transfer ID: %u\n", compl->transfer_id); + _ODP_ERR("Bad transfer ID: %u\n", compl->transfer_id); return -1; } break; case ODP_DMA_COMPL_EVENT: if (compl->event == ODP_EVENT_INVALID || compl->queue == ODP_QUEUE_INVALID) { - ODP_ERR("Bad event or queue\n"); + _ODP_ERR("Bad event or queue\n"); return -1; } break; default: - ODP_ERR("Bad completion mode %u\n", compl->compl_mode); + _ODP_ERR("Bad completion mode %u\n", compl->compl_mode); return -1; } @@ -654,7 +655,7 @@ int odp_dma_transfer_start(odp_dma_t dma, const odp_dma_transfer_param_t *transf odp_buffer_t buf = (odp_buffer_t)(uintptr_t)compl->event; if (odp_unlikely(odp_event_type(compl->event) != ODP_EVENT_DMA_COMPL)) { - ODP_ERR("Bad completion event type\n"); + _ODP_ERR("Bad completion event type\n"); return -1; } @@ -663,8 +664,8 @@ int odp_dma_transfer_start(odp_dma_t dma, const odp_dma_transfer_param_t *transf result->user_ptr = compl->user_ptr; if (odp_unlikely(odp_queue_enq(compl->queue, compl->event))) { - ODP_ERR("Completion event enqueue failed %" PRIu64 "\n", - odp_queue_to_u64(compl->queue)); + _ODP_ERR("Completion event enqueue failed %" PRIu64 "\n", + odp_queue_to_u64(compl->queue)); return -1; } } @@ -679,7 +680,7 @@ int odp_dma_transfer_start_multi(odp_dma_t dma, const odp_dma_transfer_param_t * int ret = 0; if (odp_unlikely(num < 1)) { - ODP_ERR("Bad number of transfers\n"); + _ODP_ERR("Bad number of transfers\n"); return -1; } @@ -702,13 +703,13 @@ int odp_dma_transfer_done(odp_dma_t dma, odp_dma_transfer_id_t transfer_id, dma_session_t *session = dma_session_from_handle(dma); if (odp_unlikely(dma == ODP_DMA_INVALID)) { - ODP_ERR("Bad DMA handle\n"); + _ODP_ERR("Bad DMA handle\n"); return -1; } if (odp_unlikely(transfer_id == ODP_DMA_TRANSFER_ID_INVALID || transfer_id > MAX_TRANSFERS)) { - ODP_ERR("Bad transfer ID: %u\n", transfer_id); + _ODP_ERR("Bad transfer ID: %u\n", transfer_id); return -1; } @@ -737,13 +738,13 @@ odp_pool_t odp_dma_pool_create(const char *name, const odp_dma_pool_param_t *dma uint32_t cache_size = dma_pool_param->cache_size; if (num > _odp_dma_glb->pool_capa.buf.max_num) { - ODP_ERR("Too many DMA completion events: %u\n", num); + _ODP_ERR("Too many DMA completion events: %u\n", num); return ODP_POOL_INVALID; } if (cache_size < _odp_dma_glb->pool_capa.buf.min_cache_size || cache_size > _odp_dma_glb->pool_capa.buf.max_cache_size) { - ODP_ERR("Bad cache size: %u\n", cache_size); + _ODP_ERR("Bad cache size: %u\n", cache_size); return ODP_POOL_INVALID; } @@ -784,7 +785,7 @@ void odp_dma_compl_free(odp_dma_compl_t dma_compl) odp_buffer_t buf = (odp_buffer_t)(uintptr_t)dma_compl; if (odp_unlikely(dma_compl == ODP_DMA_COMPL_INVALID)) { - ODP_ERR("Bad DMA compl handle\n"); + _ODP_ERR("Bad DMA compl handle\n"); return; } @@ -796,6 +797,8 @@ void odp_dma_compl_free(odp_dma_compl_t dma_compl) odp_dma_compl_t odp_dma_compl_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_DMA_COMPL); + return (odp_dma_compl_t)(uintptr_t)ev; } @@ -810,7 +813,7 @@ int odp_dma_compl_result(odp_dma_compl_t dma_compl, odp_dma_result_t *result_out odp_buffer_t buf = (odp_buffer_t)(uintptr_t)dma_compl; if (odp_unlikely(dma_compl == ODP_DMA_COMPL_INVALID)) { - ODP_ERR("Bad DMA compl handle\n"); + _ODP_ERR("Bad DMA compl handle\n"); return -1; } @@ -837,15 +840,15 @@ void odp_dma_print(odp_dma_t dma) dma_session_t *session = dma_session_from_handle(dma); if (dma == ODP_DMA_INVALID) { - ODP_ERR("Bad DMA handle\n"); + _ODP_ERR("Bad DMA handle\n"); return; } - ODP_PRINT("\nDMA info\n"); - ODP_PRINT("--------\n"); - ODP_PRINT(" DMA handle 0x%" PRIx64 "\n", odp_dma_to_u64(dma)); - ODP_PRINT(" name %s\n", session->name); - ODP_PRINT("\n"); + _ODP_PRINT("\nDMA info\n"); + _ODP_PRINT("--------\n"); + _ODP_PRINT(" DMA handle 0x%" PRIx64 "\n", odp_dma_to_u64(dma)); + _ODP_PRINT(" name %s\n", session->name); + _ODP_PRINT("\n"); } void odp_dma_compl_print(odp_dma_compl_t dma_compl) @@ -854,25 +857,25 @@ void odp_dma_compl_print(odp_dma_compl_t dma_compl) int ret; if (dma_compl == ODP_DMA_COMPL_INVALID) { - ODP_ERR("Bad DMA compl handle\n"); + _ODP_ERR("Bad DMA compl handle\n"); return; } ret = odp_dma_compl_result(dma_compl, &result); - ODP_PRINT("\nDMA completion\n"); - ODP_PRINT("--------------\n"); - ODP_PRINT(" Compl event handle: 0x%" PRIx64 "\n", (uint64_t)(uintptr_t)dma_compl); + _ODP_PRINT("\nDMA completion\n"); + _ODP_PRINT("--------------\n"); + _ODP_PRINT(" Compl event handle: 0x%" PRIx64 "\n", (uint64_t)(uintptr_t)dma_compl); if (ret == 0) { - ODP_PRINT(" Result: %s\n", result.success ? "success" : "fail"); - ODP_PRINT(" User pointer: 0x%" PRIx64 "\n", - (uint64_t)(uintptr_t)result.user_ptr); + _ODP_PRINT(" Result: %s\n", result.success ? "success" : "fail"); + _ODP_PRINT(" User pointer: 0x%" PRIx64 "\n", + (uint64_t)(uintptr_t)result.user_ptr); } else { - ODP_PRINT(" No result metadata\n"); + _ODP_PRINT(" No result metadata\n"); } - ODP_PRINT("\n"); + _ODP_PRINT("\n"); } int _odp_dma_init_global(void) @@ -881,7 +884,7 @@ int _odp_dma_init_global(void) int i; if (odp_global_ro.disable.dma) { - ODP_PRINT("DMA is DISABLED\n"); + _ODP_PRINT("DMA is DISABLED\n"); return 0; } @@ -889,7 +892,7 @@ int _odp_dma_init_global(void) _odp_dma_glb = odp_shm_addr(shm); if (_odp_dma_glb == NULL) { - ODP_ERR("SHM reserve failed\n"); + _ODP_ERR("SHM reserve failed\n"); return -1; } @@ -899,7 +902,7 @@ int _odp_dma_init_global(void) odp_pool_param_init(&_odp_dma_glb->pool_param); if (odp_pool_capability(&_odp_dma_glb->pool_capa)) { - ODP_ERR("Pool capability failed\n"); + _ODP_ERR("Pool capability failed\n"); return -1; } @@ -922,7 +925,7 @@ int _odp_dma_term_global(void) shm = _odp_dma_glb->shm; if (odp_shm_free(shm)) { - ODP_ERR("SHM free failed\n"); + _ODP_ERR("SHM free failed\n"); return -1; } diff --git a/platform/linux-generic/odp_errno.c b/platform/linux-generic/odp_errno.c index 71fc2da77..bce398834 100644 --- a/platform/linux-generic/odp_errno.c +++ b/platform/linux-generic/odp_errno.c @@ -24,9 +24,9 @@ void odp_errno_zero(void) void odp_errno_print(const char *str) { if (str != NULL) - ODP_PRINT("%s %s\n", str, strerror(_odp_errno)); + _ODP_PRINT("%s %s\n", str, strerror(_odp_errno)); else - ODP_PRINT("%s\n", strerror(_odp_errno)); + _ODP_PRINT("%s\n", strerror(_odp_errno)); } const char *odp_errno_str(int errnum) diff --git a/platform/linux-generic/odp_event.c b/platform/linux-generic/odp_event.c index 5a1f68621..c4e0f2c9d 100644 --- a/platform/linux-generic/odp_event.c +++ b/platform/linux-generic/odp_event.c @@ -71,7 +71,7 @@ void odp_event_free(odp_event_t event) odp_dma_compl_free(odp_dma_compl_from_event(event)); break; default: - ODP_ABORT("Invalid event type: %d\n", odp_event_type(event)); + _ODP_ABORT("Invalid event type: %d\n", odp_event_type(event)); } } diff --git a/platform/linux-generic/odp_fdserver.c b/platform/linux-generic/odp_fdserver.c index 8bb12b64a..af9c1b498 100644 --- a/platform/linux-generic/odp_fdserver.c +++ b/platform/linux-generic/odp_fdserver.c @@ -157,7 +157,7 @@ static int send_fdserver_msg(int sock, int command, } res = sendmsg(sock, &socket_message, 0); if (res < 0) { - ODP_ERR("send_fdserver_msg: %s\n", strerror(errno)); + _ODP_ERR("sendmsg() failed: %s\n", strerror(errno)); return -1; } @@ -203,7 +203,7 @@ static int recv_fdserver_msg(int sock, int *command, /* receive the message */ if (recvmsg(sock, &socket_message, MSG_CMSG_CLOEXEC) < 0) { - ODP_ERR("recv_fdserver_msg: %s\n", strerror(errno)); + _ODP_ERR("recvmsg() failed: %s\n", strerror(errno)); return -1; } @@ -246,13 +246,13 @@ static int get_socket(void) odp_global_ro.main_pid); if (len >= FDSERVER_SOCKPATH_MAXLEN || len >= (int)sizeof(remote.sun_path)) { - ODP_ERR("path too long\n"); + _ODP_ERR("path too long\n"); return -1; } s_sock = socket(AF_UNIX, SOCK_STREAM, 0); if (s_sock == -1) { - ODP_ERR("cannot connect to server: %s\n", strerror(errno)); + _ODP_ERR("cannot connect to server: %s\n", strerror(errno)); return -1; } @@ -262,7 +262,7 @@ static int get_socket(void) while (connect(s_sock, (struct sockaddr *)&remote, len) == -1) { if (errno == EINTR) continue; - ODP_ERR("cannot connect to server: %s\n", strerror(errno)); + _ODP_ERR("cannot connect to server: %s\n", strerror(errno)); close(s_sock); return -1; } @@ -292,7 +292,7 @@ int _odp_fdserver_register_fd(fd_server_context_e context, uint64_t key, res = send_fdserver_msg(s_sock, FD_REGISTER_REQ, context, key, fd_to_send); if (res < 0) { - ODP_ERR("fd registration failure\n"); + _ODP_ERR("fd registration failure\n"); close(s_sock); return -1; } @@ -300,7 +300,7 @@ int _odp_fdserver_register_fd(fd_server_context_e context, uint64_t key, res = recv_fdserver_msg(s_sock, &command, &context, &key, &fd); if ((res < 0) || (command != FD_REGISTER_ACK)) { - ODP_ERR("fd registration failure\n"); + _ODP_ERR("fd registration failure\n"); close(s_sock); return -1; } @@ -330,7 +330,7 @@ int _odp_fdserver_deregister_fd(fd_server_context_e context, uint64_t key) res = send_fdserver_msg(s_sock, FD_DEREGISTER_REQ, context, key, -1); if (res < 0) { - ODP_ERR("fd de-registration failure\n"); + _ODP_ERR("fd de-registration failure\n"); close(s_sock); return -1; } @@ -338,7 +338,7 @@ int _odp_fdserver_deregister_fd(fd_server_context_e context, uint64_t key) res = recv_fdserver_msg(s_sock, &command, &context, &key, &fd); if ((res < 0) || (command != FD_DEREGISTER_ACK)) { - ODP_ERR("fd de-registration failure\n"); + _ODP_ERR("fd de-registration failure\n"); close(s_sock); return -1; } @@ -366,7 +366,7 @@ int _odp_fdserver_lookup_fd(fd_server_context_e context, uint64_t key) res = send_fdserver_msg(s_sock, FD_LOOKUP_REQ, context, key, -1); if (res < 0) { - ODP_ERR("fd lookup failure\n"); + _ODP_ERR("fd lookup failure\n"); close(s_sock); return -1; } @@ -374,14 +374,14 @@ int _odp_fdserver_lookup_fd(fd_server_context_e context, uint64_t key) res = recv_fdserver_msg(s_sock, &command, &context, &key, &fd); if ((res < 0) || (command != FD_LOOKUP_ACK)) { - ODP_ERR("fd lookup failure\n"); + _ODP_ERR("fd lookup failure\n"); close(s_sock); return -1; } close(s_sock); - ODP_DBG("FD client lookup: pid=%d, key=%" PRIu64 ", fd=%d\n", - getpid(), key, fd); + _ODP_DBG("FD client lookup: pid=%d, key=%" PRIu64 ", fd=%d\n", + getpid(), key, fd); return fd; } @@ -402,7 +402,7 @@ static int stop_server(void) res = send_fdserver_msg(s_sock, FD_SERVERSTOP_REQ, 0, 0, -1); if (res < 0) { - ODP_ERR("fd stop request failure\n"); + _ODP_ERR("fd stop request failure\n"); close(s_sock); return -1; } @@ -430,7 +430,7 @@ static int handle_request(int client_sock) switch (command) { case FD_REGISTER_REQ: if ((fd < 0) || (context >= FD_SRV_CTX_END)) { - ODP_ERR("Invalid register fd or context\n"); + _ODP_ERR("Invalid register fd or context\n"); send_fdserver_msg(client_sock, FD_REGISTER_NACK, FD_SRV_CTX_NA, 0, -1); return 0; @@ -444,7 +444,7 @@ static int handle_request(int client_sock) ODP_DBG_LVL(FD_DBG, "storing {ctx=%d, key=%" PRIu64 "}->fd=%d\n", context, key, fd); } else { - ODP_ERR("FD table full\n"); + _ODP_ERR("FD table full\n"); send_fdserver_msg(client_sock, FD_REGISTER_NACK, FD_SRV_CTX_NA, 0, -1); return 0; @@ -456,7 +456,7 @@ static int handle_request(int client_sock) case FD_LOOKUP_REQ: if (context >= FD_SRV_CTX_END) { - ODP_ERR("invalid lookup context\n"); + _ODP_ERR("invalid lookup context\n"); send_fdserver_msg(client_sock, FD_LOOKUP_NACK, FD_SRV_CTX_NA, 0, -1); return 0; @@ -467,7 +467,7 @@ static int handle_request(int client_sock) if ((fd_table[i].context == context) && (fd_table[i].key == key)) { fd = fd_table[i].fd; - ODP_DBG("lookup {ctx=%d," + _ODP_DBG("lookup {ctx=%d," " key=%" PRIu64 "}->fd=%d\n", context, key, fd); send_fdserver_msg(client_sock, @@ -484,7 +484,7 @@ static int handle_request(int client_sock) case FD_DEREGISTER_REQ: if (context >= FD_SRV_CTX_END) { - ODP_ERR("invalid deregister context\n"); + _ODP_ERR("invalid deregister context\n"); send_fdserver_msg(client_sock, FD_DEREGISTER_NACK, FD_SRV_CTX_NA, 0, -1); return 0; @@ -516,7 +516,7 @@ static int handle_request(int client_sock) return 1; default: - ODP_ERR("Unexpected request\n"); + _ODP_ERR("Unexpected request\n"); break; } return 0; @@ -539,7 +539,7 @@ static void wait_requests(int sock) if (errno == EINTR) continue; - ODP_ERR("wait_requests: %s\n", strerror(errno)); + _ODP_ERR("accept() failed: %s\n", strerror(errno)); return; } @@ -574,14 +574,14 @@ int _odp_fdserver_init_global(void) odp_global_ro.main_pid); if (len >= FDSERVER_SOCKPATH_MAXLEN || len >= (int)sizeof(local.sun_path)) { - ODP_ERR("path too long\n"); + _ODP_ERR("path too long\n"); return -1; } /* create UNIX domain socket: */ sock = socket(AF_UNIX, SOCK_STREAM, 0); if (sock == -1) { - ODP_ERR("_odp_fdserver_init_global: %s\n", strerror(errno)); + _ODP_ERR("socket() failed: %s\n", strerror(errno)); return -1; } @@ -595,14 +595,14 @@ int _odp_fdserver_init_global(void) res = bind(sock, (struct sockaddr *)&local, sizeof(struct sockaddr_un)); if (res == -1) { - ODP_ERR("_odp_fdserver_init_global: %s\n", strerror(errno)); + _ODP_ERR("bind() failed: %s\n", strerror(errno)); close(sock); return -1; } - /* listen for incoming conections: */ + /* listen for incoming connections: */ if (listen(sock, FDSERVER_BACKLOG) == -1) { - ODP_ERR("_odp_fdserver_init_global: %s\n", strerror(errno)); + _ODP_ERR("listen() failed: %s\n", strerror(errno)); close(sock); return -1; } @@ -610,7 +610,7 @@ int _odp_fdserver_init_global(void) /* fork a server process: */ server_pid = fork(); if (server_pid == -1) { - ODP_ERR("Could not fork!\n"); + _ODP_ERR("Could not fork!\n"); close(sock); return -1; } @@ -630,7 +630,7 @@ int _odp_fdserver_init_global(void) /* these we want to handle */ sigdelset(&sigset, SIGTERM); if (sigprocmask(SIG_SETMASK, &sigset, NULL) == -1) { - ODP_ERR("Could not set signal mask"); + _ODP_ERR("Could not set signal mask"); exit(1); } @@ -653,14 +653,14 @@ int _odp_fdserver_init_global(void) res = setsid(); if (res == -1) { - ODP_ERR("Could not setsid()"); + _ODP_ERR("Could not setsid()"); exit(1); } /* allocate the space for the file descriptor<->key table: */ fd_table = malloc(FDSERVER_MAX_ENTRIES * sizeof(fdentry_t)); if (!fd_table) { - ODP_ERR("maloc failed!\n"); + _ODP_ERR("maloc failed!\n"); exit(1); } @@ -691,15 +691,15 @@ int _odp_fdserver_term_global(void) /* close fdserver and wait for it to terminate */ if (stop_server()) { - ODP_ERR("Server stop failed\n"); + _ODP_ERR("Server stop failed\n"); return -1; } - ODP_DBG("Waiting for fdserver (%i) to stop\n", odp_global_ro.fdserver_pid); + _ODP_DBG("Waiting for fdserver (%i) to stop\n", odp_global_ro.fdserver_pid); pid = waitpid(odp_global_ro.fdserver_pid, &status, 0); if (pid != odp_global_ro.fdserver_pid) - ODP_ERR("Failed to wait for fdserver\n"); + _ODP_ERR("Failed to wait for fdserver\n"); /* construct the server named socket path: */ snprintf(sockpath, FDSERVER_SOCKPATH_MAXLEN, FDSERVER_SOCK_FORMAT, diff --git a/platform/linux-generic/odp_hash_crc_gen.c b/platform/linux-generic/odp_hash_crc_gen.c index 109d15420..f831c63b8 100644 --- a/platform/linux-generic/odp_hash_crc_gen.c +++ b/platform/linux-generic/odp_hash_crc_gen.c @@ -39,7 +39,7 @@ int _odp_hash_init_global(void) crc_table = odp_shm_addr(shm); if (crc_table == NULL) { - ODP_ERR("Shm reserve failed for odp_hash_crc_gen\n"); + _ODP_ERR("Shm reserve failed for odp_hash_crc_gen\n"); return -1; } @@ -54,7 +54,7 @@ int _odp_hash_init_global(void) int _odp_hash_term_global(void) { if (odp_shm_free(crc_table->shm)) { - ODP_ERR("Shm free failed for odp_hash_crc_gen\n"); + _ODP_ERR("Shm free failed for odp_hash_crc_gen\n"); return -1; } @@ -203,18 +203,18 @@ int odp_hash_crc_gen64(const void *data_ptr, uint32_t data_len, int reflect = crc_param->reflect_in; if (odp_unlikely(crc_param->reflect_in != crc_param->reflect_out)) { - ODP_ERR("Odd reflection setting not supported.\n"); + _ODP_ERR("Odd reflection setting not supported.\n"); return -1; } if (odp_unlikely(width != 32 && width != 24 && width != 16)) { - ODP_ERR("CRC width %" PRIu32 " bits not supported.\n", width); + _ODP_ERR("CRC width %" PRIu32 " bits not supported.\n", width); return -1; } /* TODO: fix implementation of 24 bit CRC with reflection */ if (odp_unlikely(width == 24 && reflect)) { - ODP_ERR("24 bit CRC with reflection not supported.\n"); + _ODP_ERR("24 bit CRC with reflection not supported.\n"); return -1; } diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index 0c49946b0..284b3e566 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -9,11 +9,15 @@ #include <odp/api/init.h> #include <odp/api/shared_memory.h> + +#include <odp/api/plat/thread_inlines.h> + #include <odp_debug_internal.h> +#include <odp_global_data.h> #include <odp_init_internal.h> #include <odp_schedule_if.h> #include <odp_libconfig_internal.h> -#include <odp/api/plat/thread_inlines.h> + #include <string.h> #include <stdio.h> #include <unistd.h> @@ -52,6 +56,15 @@ enum init_stage { odp_global_data_ro_t odp_global_ro; odp_global_data_rw_t *odp_global_rw; +/* Global function pointers for inline header usage. The values are written + * during odp_init_global() (enables process mode support). */ +#include <odp/visibility_begin.h> + +odp_log_func_t ODP_PRINTF_FORMAT(2, 3) _odp_log_fn; +odp_abort_func_t _odp_abort_fn; + +#include <odp/visibility_end.h> + /* odp_init_local() call status */ static __thread uint8_t init_local_called; @@ -98,7 +111,7 @@ static int global_rw_data_init(void) odp_global_rw = odp_shm_addr(shm); if (odp_global_rw == NULL) { - ODP_ERR("Global RW data shm reserve failed.\n"); + _ODP_ERR("Global RW data shm reserve failed.\n"); return -1; } @@ -113,12 +126,12 @@ static int global_rw_data_term(void) shm = odp_shm_lookup("_odp_global_rw_data"); if (shm == ODP_SHM_INVALID) { - ODP_ERR("Unable to find global RW data shm.\n"); + _ODP_ERR("Unable to find global RW data shm.\n"); return -1; } if (odp_shm_free(shm)) { - ODP_ERR("Global RW data shm free failed.\n"); + _ODP_ERR("Global RW data shm free failed.\n"); return -1; } @@ -133,154 +146,154 @@ static int term_global(enum init_stage stage) case ALL_INIT: case DMA_INIT: if (_odp_dma_term_global()) { - ODP_ERR("ODP DMA term failed.\n"); + _ODP_ERR("ODP DMA term failed.\n"); rc = -1; } /* Fall through */ case IPSEC_INIT: if (_odp_ipsec_term_global()) { - ODP_ERR("ODP IPsec term failed.\n"); + _ODP_ERR("ODP IPsec term failed.\n"); rc = -1; } /* Fall through */ case IPSEC_SAD_INIT: if (_odp_ipsec_sad_term_global()) { - ODP_ERR("ODP IPsec SAD term failed.\n"); + _ODP_ERR("ODP IPsec SAD term failed.\n"); rc = -1; } /* Fall through */ case IPSEC_EVENTS_INIT: if (_odp_ipsec_events_term_global()) { - ODP_ERR("ODP IPsec events term failed.\n"); + _ODP_ERR("ODP IPsec events term failed.\n"); rc = -1; } /* Fall through */ case NAME_TABLE_INIT: if (_odp_int_name_tbl_term_global()) { - ODP_ERR("Name table term failed.\n"); + _ODP_ERR("Name table term failed.\n"); rc = -1; } /* Fall through */ case TRAFFIC_MNGR_INIT: if (_odp_tm_term_global()) { - ODP_ERR("TM term failed.\n"); + _ODP_ERR("TM term failed.\n"); rc = -1; } /* Fall through */ case CLASSIFICATION_INIT: if (_odp_classification_term_global()) { - ODP_ERR("ODP classification term failed.\n"); + _ODP_ERR("ODP classification term failed.\n"); rc = -1; } /* Fall through */ case COMP_INIT: if (_odp_comp_term_global()) { - ODP_ERR("ODP comp term failed.\n"); + _ODP_ERR("ODP comp term failed.\n"); rc = -1; } /* Fall through */ case CRYPTO_INIT: if (_odp_crypto_term_global()) { - ODP_ERR("ODP crypto term failed.\n"); + _ODP_ERR("ODP crypto term failed.\n"); rc = -1; } /* Fall through */ case TIMER_INIT: if (_odp_timer_term_global()) { - ODP_ERR("ODP timer term failed.\n"); + _ODP_ERR("ODP timer term failed.\n"); rc = -1; } /* Fall through */ case PKTIO_INIT: if (_odp_pktio_term_global()) { - ODP_ERR("ODP pktio term failed.\n"); + _ODP_ERR("ODP pktio term failed.\n"); rc = -1; } /* Fall through */ case SCHED_INIT: if (_odp_schedule_term_global()) { - ODP_ERR("ODP schedule term failed.\n"); + _ODP_ERR("ODP schedule term failed.\n"); rc = -1; } /* Fall through */ case QUEUE_INIT: if (_odp_queue_term_global()) { - ODP_ERR("ODP queue term failed.\n"); + _ODP_ERR("ODP queue term failed.\n"); rc = -1; } /* Fall through */ case STASH_INIT: if (_odp_stash_term_global()) { - ODP_ERR("ODP stash term failed.\n"); + _ODP_ERR("ODP stash term failed.\n"); rc = -1; } /* Fall through */ case POOL_INIT: if (_odp_pool_term_global()) { - ODP_ERR("ODP buffer pool term failed.\n"); + _ODP_ERR("ODP buffer pool term failed.\n"); rc = -1; } /* Fall through */ case THREAD_INIT: if (_odp_thread_term_global()) { - ODP_ERR("ODP thread term failed.\n"); + _ODP_ERR("ODP thread term failed.\n"); rc = -1; } /* Fall through */ case HASH_INIT: if (_odp_hash_term_global()) { - ODP_ERR("ODP hash term failed.\n"); + _ODP_ERR("ODP hash term failed.\n"); rc = -1; } /* Fall through */ case GLOBAL_RW_DATA_INIT: if (global_rw_data_term()) { - ODP_ERR("ODP global RW data term failed.\n"); + _ODP_ERR("ODP global RW data term failed.\n"); rc = -1; } /* Fall through */ case FDSERVER_INIT: if (_odp_fdserver_term_global()) { - ODP_ERR("ODP fdserver term failed.\n"); + _ODP_ERR("ODP fdserver term failed.\n"); rc = -1; } /* Fall through */ case ISHM_INIT: if (_odp_ishm_term_global()) { - ODP_ERR("ODP ishm term failed.\n"); + _ODP_ERR("ODP ishm term failed.\n"); rc = -1; } /* Fall through */ case SYSINFO_INIT: if (_odp_system_info_term()) { - ODP_ERR("ODP system info term failed.\n"); + _ODP_ERR("ODP system info term failed.\n"); rc = -1; } /* Fall through */ case TIME_INIT: if (_odp_time_term_global()) { - ODP_ERR("ODP time term failed.\n"); + _ODP_ERR("ODP time term failed.\n"); rc = -1; } /* Fall through */ @@ -289,14 +302,14 @@ static int term_global(enum init_stage stage) /* Fall through */ case CPUMASK_INIT: if (_odp_cpumask_term_global()) { - ODP_ERR("ODP cpumask term failed.\n"); + _ODP_ERR("ODP cpumask term failed.\n"); rc = -1; } /* Fall through */ case LIBCONFIG_INIT: if (_odp_libconfig_term_global()) { - ODP_ERR("ODP runtime config term failed.\n"); + _ODP_ERR("ODP runtime config term failed.\n"); rc = -1; } /* Fall through */ @@ -316,23 +329,23 @@ int odp_init_global(odp_instance_t *instance, memset(&odp_global_ro, 0, sizeof(odp_global_data_ro_t)); odp_global_ro.main_pid = getpid(); - odp_global_ro.log_fn = odp_override_log; - odp_global_ro.abort_fn = odp_override_abort; + _odp_log_fn = odp_override_log; + _odp_abort_fn = odp_override_abort; odp_init_param_init(&odp_global_ro.init_param); if (params != NULL) { odp_global_ro.init_param = *params; if (params->log_fn != NULL) - odp_global_ro.log_fn = params->log_fn; + _odp_log_fn = params->log_fn; if (params->abort_fn != NULL) - odp_global_ro.abort_fn = params->abort_fn; + _odp_abort_fn = params->abort_fn; if (params->mem_model == ODP_MEM_MODEL_PROCESS) odp_global_ro.shm_single_va = 1; } if (_odp_libconfig_init_global()) { - ODP_ERR("ODP runtime config init failed.\n"); + _ODP_ERR("ODP runtime config init failed.\n"); goto init_failed; } stage = LIBCONFIG_INIT; @@ -340,145 +353,145 @@ int odp_init_global(odp_instance_t *instance, disable_features(&odp_global_ro, params); if (_odp_cpumask_init_global(params)) { - ODP_ERR("ODP cpumask init failed.\n"); + _ODP_ERR("ODP cpumask init failed.\n"); goto init_failed; } stage = CPUMASK_INIT; if (_odp_cpu_cycles_init_global()) { - ODP_ERR("ODP cpu cycle init failed.\n"); + _ODP_ERR("ODP cpu cycle init failed.\n"); goto init_failed; } stage = CPU_CYCLES_INIT; if (_odp_time_init_global()) { - ODP_ERR("ODP time init failed.\n"); + _ODP_ERR("ODP time init failed.\n"); goto init_failed; } stage = TIME_INIT; if (_odp_system_info_init()) { - ODP_ERR("ODP system_info init failed.\n"); + _ODP_ERR("ODP system_info init failed.\n"); goto init_failed; } stage = SYSINFO_INIT; if (_odp_ishm_init_global(params)) { - ODP_ERR("ODP ishm init failed.\n"); + _ODP_ERR("ODP ishm init failed.\n"); goto init_failed; } stage = ISHM_INIT; if (_odp_fdserver_init_global()) { - ODP_ERR("ODP fdserver init failed.\n"); + _ODP_ERR("ODP fdserver init failed.\n"); goto init_failed; } stage = FDSERVER_INIT; if (global_rw_data_init()) { - ODP_ERR("ODP global RW data init failed.\n"); + _ODP_ERR("ODP global RW data init failed.\n"); goto init_failed; } stage = GLOBAL_RW_DATA_INIT; if (_odp_hash_init_global()) { - ODP_ERR("ODP hash init failed.\n"); + _ODP_ERR("ODP hash init failed.\n"); goto init_failed; } stage = HASH_INIT; if (_odp_thread_init_global()) { - ODP_ERR("ODP thread init failed.\n"); + _ODP_ERR("ODP thread init failed.\n"); goto init_failed; } stage = THREAD_INIT; if (_odp_pool_init_global()) { - ODP_ERR("ODP pool init failed.\n"); + _ODP_ERR("ODP pool init failed.\n"); goto init_failed; } stage = POOL_INIT; if (_odp_stash_init_global()) { - ODP_ERR("ODP stash init failed.\n"); + _ODP_ERR("ODP stash init failed.\n"); goto init_failed; } stage = STASH_INIT; if (_odp_queue_init_global()) { - ODP_ERR("ODP queue init failed.\n"); + _ODP_ERR("ODP queue init failed.\n"); goto init_failed; } stage = QUEUE_INIT; if (_odp_schedule_init_global()) { - ODP_ERR("ODP schedule init failed.\n"); + _ODP_ERR("ODP schedule init failed.\n"); goto init_failed; } stage = SCHED_INIT; if (_odp_pktio_init_global()) { - ODP_ERR("ODP packet io init failed.\n"); + _ODP_ERR("ODP packet io init failed.\n"); goto init_failed; } stage = PKTIO_INIT; if (_odp_timer_init_global(params)) { - ODP_ERR("ODP timer init failed.\n"); + _ODP_ERR("ODP timer init failed.\n"); goto init_failed; } stage = TIMER_INIT; if (_odp_crypto_init_global()) { - ODP_ERR("ODP crypto init failed.\n"); + _ODP_ERR("ODP crypto init failed.\n"); goto init_failed; } stage = CRYPTO_INIT; if (_odp_comp_init_global()) { - ODP_ERR("ODP comp init failed.\n"); + _ODP_ERR("ODP comp init failed.\n"); goto init_failed; } stage = COMP_INIT; if (_odp_classification_init_global()) { - ODP_ERR("ODP classification init failed.\n"); + _ODP_ERR("ODP classification init failed.\n"); goto init_failed; } stage = CLASSIFICATION_INIT; if (_odp_tm_init_global()) { - ODP_ERR("ODP traffic manager init failed\n"); + _ODP_ERR("ODP traffic manager init failed\n"); goto init_failed; } stage = TRAFFIC_MNGR_INIT; if (_odp_int_name_tbl_init_global()) { - ODP_ERR("ODP name table init failed\n"); + _ODP_ERR("ODP name table init failed\n"); goto init_failed; } stage = NAME_TABLE_INIT; if (_odp_ipsec_events_init_global()) { - ODP_ERR("ODP IPsec events init failed.\n"); + _ODP_ERR("ODP IPsec events init failed.\n"); goto init_failed; } stage = IPSEC_EVENTS_INIT; if (_odp_ipsec_sad_init_global()) { - ODP_ERR("ODP IPsec SAD init failed.\n"); + _ODP_ERR("ODP IPsec SAD init failed.\n"); goto init_failed; } stage = IPSEC_SAD_INIT; if (_odp_ipsec_init_global()) { - ODP_ERR("ODP IPsec init failed.\n"); + _ODP_ERR("ODP IPsec init failed.\n"); goto init_failed; } stage = IPSEC_INIT; if (_odp_dma_init_global()) { - ODP_ERR("ODP DMA init failed.\n"); + _ODP_ERR("ODP DMA init failed.\n"); goto init_failed; } stage = DMA_INIT; @@ -495,7 +508,7 @@ init_failed: int odp_term_global(odp_instance_t instance) { if (instance != (odp_instance_t)odp_global_ro.main_pid) { - ODP_ERR("Bad instance.\n"); + _ODP_ERR("Bad instance.\n"); return -1; } return term_global(ALL_INIT); @@ -511,42 +524,42 @@ static int term_local(enum init_stage stage) case SCHED_INIT: if (_odp_sched_fn->term_local()) { - ODP_ERR("ODP schedule local term failed.\n"); + _ODP_ERR("ODP schedule local term failed.\n"); rc = -1; } /* Fall through */ case QUEUE_INIT: if (_odp_queue_fn->term_local()) { - ODP_ERR("ODP queue local term failed.\n"); + _ODP_ERR("ODP queue local term failed.\n"); rc = -1; } /* Fall through */ case POOL_INIT: if (_odp_pool_term_local()) { - ODP_ERR("ODP buffer pool local term failed.\n"); + _ODP_ERR("ODP buffer pool local term failed.\n"); rc = -1; } /* Fall through */ case CRYPTO_INIT: if (_odp_crypto_term_local()) { - ODP_ERR("ODP crypto local term failed.\n"); + _ODP_ERR("ODP crypto local term failed.\n"); rc = -1; } /* Fall through */ case RANDOM_INIT: if (_odp_random_term_local()) { - ODP_ERR("ODP random local term failed.\n"); + _ODP_ERR("ODP random local term failed.\n"); rc = -1; } /* Fall through */ case TIMER_INIT: if (_odp_timer_term_local()) { - ODP_ERR("ODP timer local term failed.\n"); + _ODP_ERR("ODP timer local term failed.\n"); rc = -1; } /* Fall through */ @@ -554,7 +567,7 @@ static int term_local(enum init_stage stage) case THREAD_INIT: rc_thd = _odp_thread_term_local(); if (rc_thd < 0) { - ODP_ERR("ODP thread local term failed.\n"); + _ODP_ERR("ODP thread local term failed.\n"); rc = -1; } else { if (!rc) @@ -564,7 +577,7 @@ static int term_local(enum init_stage stage) case ISHM_INIT: if (_odp_ishm_term_local()) { - ODP_ERR("ODP ishm local term failed.\n"); + _ODP_ERR("ODP ishm local term failed.\n"); rc = -1; } /* Fall through */ @@ -581,67 +594,67 @@ int odp_init_local(odp_instance_t instance, odp_thread_type_t thr_type) enum init_stage stage = NO_INIT; if (instance != (odp_instance_t)odp_global_ro.main_pid) { - ODP_ERR("Bad instance.\n"); + _ODP_ERR("Bad instance.\n"); goto init_fail; } /* Detect if odp_init_local() has been already called from this thread */ if (getpid() == odp_global_ro.main_pid && init_local_called) { - ODP_ERR("%s() called multiple times by the same thread\n", __func__); + _ODP_ERR("%s() called multiple times by the same thread\n", __func__); goto init_fail; } init_local_called = 1; if (_odp_ishm_init_local()) { - ODP_ERR("ODP ishm local init failed.\n"); + _ODP_ERR("ODP ishm local init failed.\n"); goto init_fail; } stage = ISHM_INIT; if (_odp_thread_init_local(thr_type)) { - ODP_ERR("ODP thread local init failed.\n"); + _ODP_ERR("ODP thread local init failed.\n"); goto init_fail; } stage = THREAD_INIT; if (_odp_pktio_init_local()) { - ODP_ERR("ODP packet io local init failed.\n"); + _ODP_ERR("ODP packet io local init failed.\n"); goto init_fail; } stage = PKTIO_INIT; if (_odp_timer_init_local()) { - ODP_ERR("ODP timer local init failed.\n"); + _ODP_ERR("ODP timer local init failed.\n"); goto init_fail; } stage = TIMER_INIT; if (_odp_random_init_local()) { - ODP_ERR("ODP random local init failed.\n"); + _ODP_ERR("ODP random local init failed.\n"); goto init_fail; } stage = RANDOM_INIT; if (_odp_crypto_init_local()) { - ODP_ERR("ODP crypto local init failed.\n"); + _ODP_ERR("ODP crypto local init failed.\n"); goto init_fail; } stage = CRYPTO_INIT; if (_odp_pool_init_local()) { - ODP_ERR("ODP pool local init failed.\n"); + _ODP_ERR("ODP pool local init failed.\n"); goto init_fail; } stage = POOL_INIT; if (_odp_queue_fn->init_local()) { - ODP_ERR("ODP queue local init failed.\n"); + _ODP_ERR("ODP queue local init failed.\n"); goto init_fail; } stage = QUEUE_INIT; if (_odp_sched_fn->init_local()) { - ODP_ERR("ODP schedule local init failed.\n"); + _ODP_ERR("ODP schedule local init failed.\n"); goto init_fail; } /* stage = SCHED_INIT; */ @@ -657,7 +670,7 @@ int odp_term_local(void) { /* Check that odp_init_local() has been called by this thread */ if (!init_local_called) { - ODP_ERR("%s() called by a non-initialized thread\n", __func__); + _ODP_ERR("%s() called by a non-initialized thread\n", __func__); return -1; } init_local_called = 0; diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 98fd2ac53..6be1a01ad 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -146,7 +146,7 @@ int odp_ipsec_capability(odp_ipsec_capability_t *capa) odp_queue_capability_t queue_capa; if (odp_global_ro.disable.ipsec) { - ODP_ERR("IPSec is disabled\n"); + _ODP_ERR("IPSec is disabled\n"); return -1; } @@ -315,7 +315,7 @@ odp_bool_t _odp_ipsec_is_sync_mode(odp_ipsec_dir_t dir) static odp_ipsec_packet_result_t *ipsec_pkt_result(odp_packet_t packet) { - ODP_ASSERT(ODP_EVENT_PACKET_IPSEC == + _ODP_ASSERT(ODP_EVENT_PACKET_IPSEC == odp_event_subtype(odp_packet_to_event(packet))); return &packet_hdr(packet)->ipsec_ctx; @@ -403,6 +403,23 @@ typedef struct { uint8_t iv[IPSEC_MAX_IV_LEN]; } ipsec_state_t; +#define MAX_BURST 32 + +typedef struct { + ipsec_state_t state; + odp_ipsec_op_status_t status; + ipsec_sa_t *sa; + odp_ipsec_sa_t sa_hdl; + uint32_t orig_ip_len; +} ipsec_op_t; + +#define MAX_HDR_LEN 100 /* Enough for VxLAN over IPv6 */ + +typedef struct { + ipsec_op_t op; + uint8_t hdr_buf[MAX_HDR_LEN]; +} ipsec_inline_op_t; + /* * Computes 64-bit seq number according to RFC4303 A2 */ @@ -517,7 +534,7 @@ static inline ipsec_sa_t *ipsec_get_sa(odp_ipsec_sa_t sa, } } else { ipsec_sa = _odp_ipsec_sa_entry_from_hdl(sa); - ODP_ASSERT(NULL != ipsec_sa); + _ODP_ASSERT(NULL != ipsec_sa); if (ipsec_sa->proto != proto || ipsec_sa->spi != spi) { status->error.proto = 1; @@ -646,7 +663,7 @@ static int ipsec_in_esp(odp_packet_t *pkt, if (odp_packet_extend_tail(pkt, IPSEC_SEQ_HI_LEN, NULL, NULL) < 0) { status->error.alg = 1; - ODP_ERR("odp_packet_extend_tail failed\n"); + _ODP_ERR("odp_packet_extend_tail failed\n"); return -1; } odp_packet_move_data(*pkt, icv_offset + IPSEC_SEQ_HI_LEN, icv_offset, @@ -665,6 +682,7 @@ static int ipsec_in_esp(odp_packet_t *pkt, ipsec_sa->icv_len; state->stats_length = param->cipher_range.length; + param->session = ipsec_sa->session; return 0; } @@ -775,7 +793,7 @@ static int ipsec_in_ah(odp_packet_t *pkt, if (odp_packet_extend_tail(pkt, IPSEC_SEQ_HI_LEN, NULL, NULL) < 0) { status->error.alg = 1; - ODP_ERR("odp_packet_extend_tail failed\n"); + _ODP_ERR("odp_packet_extend_tail failed\n"); return -1; } odp_packet_copy_from_mem(*pkt, seqh_offset, IPSEC_SEQ_HI_LEN, &inb_seqh); @@ -788,6 +806,7 @@ static int ipsec_in_ah(odp_packet_t *pkt, state->stats_length = param->auth_range.length; param->auth_range.length += ipsec_get_seqh_len(ipsec_sa); + param->session = ipsec_sa->session; return 0; } @@ -860,220 +879,308 @@ ipsec_sa_err_stats_update(ipsec_sa_t *sa, odp_ipsec_op_status_t *status) odp_atomic_inc_u64(&sa->stats.hard_exp_pkts_err); } -static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, - odp_ipsec_sa_t sa, - odp_packet_t *pkt_out, - odp_bool_t enqueue_op, - odp_ipsec_op_status_t *status, - uint32_t *orig_ip_len) +static int ipsec_in_parse_encap_packet(odp_packet_t pkt, ipsec_state_t *state, + odp_ipsec_op_status_t *status, uint32_t *orig_ip_len) { - ipsec_state_t state; - ipsec_sa_t *ipsec_sa = NULL; - odp_crypto_packet_op_param_t param; - int rc; - odp_crypto_packet_result_t crypto; /**< Crypto operation result */ + int (*op)(ipsec_state_t *state, odp_packet_t pkt); - state.ip_offset = odp_packet_l3_offset(pkt); - ODP_ASSERT(ODP_PACKET_OFFSET_INVALID != state.ip_offset); + state->ip_offset = odp_packet_l3_offset(pkt); + _ODP_ASSERT(ODP_PACKET_OFFSET_INVALID != state->ip_offset); + state->ip = odp_packet_l3_ptr(pkt, NULL); + _ODP_ASSERT(NULL != state->ip); + state->is_ipv4 = (((uint8_t *)state->ip)[0] >> 4) == 0x4; + state->is_ipv6 = (((uint8_t *)state->ip)[0] >> 4) == 0x6; - state.ip = odp_packet_l3_ptr(pkt, NULL); - ODP_ASSERT(NULL != state.ip); + if (odp_unlikely(!(state->is_ipv4 || state->is_ipv6))) + goto err; - /* Initialize parameters block */ - memset(¶m, 0, sizeof(param)); + op = state->is_ipv4 ? ipsec_parse_ipv4 : ipsec_parse_ipv6; - /* - * FIXME: maybe use packet flag as below ??? - * This adds requirement that input packets contain not only valid - * l3/l4 offsets, but also valid packet flags - * state.is_ipv4 = odp_packet_has_ipv4(pkt); - */ - state.is_ipv4 = (((uint8_t *)state.ip)[0] >> 4) == 0x4; - state.is_ipv6 = (((uint8_t *)state.ip)[0] >> 4) == 0x6; - if (state.is_ipv4) - rc = ipsec_parse_ipv4(&state, pkt); - else if (state.is_ipv6) - rc = ipsec_parse_ipv6(&state, pkt); - else - rc = -1; - if (rc < 0 || - state.ip_tot_len + state.ip_offset > odp_packet_len(pkt)) { - status->error.alg = 1; - goto exit; - } - *orig_ip_len = state.ip_tot_len; + if (odp_unlikely(op(state, pkt) || + state->ip_tot_len + state->ip_offset > odp_packet_len(pkt))) + goto err; - /* Check IP header for IPSec protocols and look it up */ - if (_ODP_IPPROTO_ESP == state.ip_next_hdr || - _ODP_IPPROTO_UDP == state.ip_next_hdr) { - rc = ipsec_in_esp(&pkt, &state, &ipsec_sa, sa, ¶m, status); - } else if (_ODP_IPPROTO_AH == state.ip_next_hdr) { - rc = ipsec_in_ah(&pkt, &state, &ipsec_sa, sa, ¶m, status); - } else { + *orig_ip_len = state->ip_tot_len; + + return 0; + +err: + status->error.alg = 1; + + return -1; +} + +static int ipsec_in_prepare_op(odp_packet_t *pkt, ipsec_state_t *state, ipsec_sa_t **ipsec_sa, + odp_ipsec_sa_t sa, odp_crypto_packet_op_param_t *param, + odp_ipsec_op_status_t *status) +{ + int (*op)(odp_packet_t *pkt, ipsec_state_t *state, ipsec_sa_t **ipsec_sa, + odp_ipsec_sa_t sa, odp_crypto_packet_op_param_t *param, + odp_ipsec_op_status_t *status); + + memset(param, 0, sizeof(*param)); + + if (odp_unlikely(!(_ODP_IPPROTO_ESP == state->ip_next_hdr || + _ODP_IPPROTO_UDP == state->ip_next_hdr || + _ODP_IPPROTO_AH == state->ip_next_hdr))) { status->error.proto = 1; - goto exit; + + return -1; } - if (rc < 0) - goto exit; - if (_odp_ipsec_sa_replay_precheck(ipsec_sa, - state.in.seq_no, - status) < 0) - goto exit; + op = _ODP_IPPROTO_ESP == state->ip_next_hdr || _ODP_IPPROTO_UDP == state->ip_next_hdr ? + ipsec_in_esp : ipsec_in_ah; - if (_odp_ipsec_sa_stats_precheck(ipsec_sa, status) < 0) - goto exit; + return op(pkt, state, ipsec_sa, sa, param, status); +} - param.session = ipsec_sa->session; +static int ipsec_in_prepare_packet(odp_packet_t *pkt, ipsec_state_t *state, ipsec_sa_t **ipsec_sa, + odp_ipsec_sa_t sa, odp_crypto_packet_op_param_t *param, + odp_ipsec_op_status_t *status, uint32_t *orig_ip_len) +{ + return ipsec_in_parse_encap_packet(*pkt, state, status, orig_ip_len) || + ipsec_in_prepare_op(pkt, state, ipsec_sa, sa, param, status) || + _odp_ipsec_sa_replay_precheck(*ipsec_sa, state->in.seq_no, status) < 0 || + _odp_ipsec_sa_stats_precheck(*ipsec_sa, status) < 0; +} - rc = odp_crypto_op(&pkt, &pkt, ¶m, 1); - if (rc < 0) { - ODP_DBG("Crypto failed\n"); - status->error.alg = 1; - goto exit; +static int ipsec_in_do_crypto(odp_packet_t *pkt, odp_crypto_packet_op_param_t *param, + odp_ipsec_op_status_t *status) +{ + odp_crypto_packet_result_t crypto; + + if (odp_unlikely(odp_crypto_op(pkt, pkt, param, 1) < 0)) { + _ODP_DBG("Crypto failed\n"); + goto alg_err; } - rc = odp_crypto_result(&crypto, pkt); - if (rc < 0) { - ODP_DBG("Crypto failed\n"); - status->error.alg = 1; - goto exit; + if (odp_unlikely(odp_crypto_result(&crypto, *pkt) < 0)) { + _ODP_DBG("Crypto failed\n"); + goto alg_err; } - if (!crypto.ok) { - if ((crypto.cipher_status.alg_err == - ODP_CRYPTO_ALG_ERR_ICV_CHECK) || - (crypto.auth_status.alg_err == - ODP_CRYPTO_ALG_ERR_ICV_CHECK)) - status->error.auth = 1; + if (odp_unlikely(!crypto.ok)) { + if (crypto.cipher_status.alg_err == ODP_CRYPTO_ALG_ERR_ICV_CHECK || + crypto.auth_status.alg_err == ODP_CRYPTO_ALG_ERR_ICV_CHECK) + goto auth_err; else - status->error.alg = 1; - - goto exit; + goto alg_err; } - if (ipsec_sa->antireplay) { - if (enqueue_op) - wait_for_order(ipsec_global->inbound_ordering_mode); + return 0; - if (_odp_ipsec_sa_replay_update(ipsec_sa, - state.in.seq_no, - status) < 0) - goto exit; - } +alg_err: + status->error.alg = 1; - if (_odp_ipsec_sa_lifetime_update(ipsec_sa, - state.stats_length, - status) < 0) - goto post_lifetime_err_cnt_update; + return -1; - state.ip = odp_packet_l3_ptr(pkt, NULL); +auth_err: + status->error.auth = 1; - if (ODP_IPSEC_ESP == ipsec_sa->proto) - rc = ipsec_in_esp_post(pkt, &state); - else if (ODP_IPSEC_AH == ipsec_sa->proto) - rc = ipsec_in_ah_post(pkt, &state); - else - rc = -1; - if (rc < 0) { + return -1; +} + +static int ipsec_in_finalize_op(odp_packet_t *pkt, ipsec_state_t *state, ipsec_sa_t *ipsec_sa, + odp_ipsec_op_status_t *status) +{ + int (*op)(odp_packet_t pkt, ipsec_state_t *state); + + state->ip = odp_packet_l3_ptr(*pkt, NULL); + + if (odp_unlikely(!(ODP_IPSEC_ESP == ipsec_sa->proto || ODP_IPSEC_AH == ipsec_sa->proto))) + goto proto_err; + + op = ODP_IPSEC_ESP == ipsec_sa->proto ? ipsec_in_esp_post : ipsec_in_ah_post; + + if (odp_unlikely(op(*pkt, state))) + goto proto_err; + + if (odp_unlikely(odp_packet_trunc_tail(pkt, + state->in.trl_len + ipsec_get_seqh_len(ipsec_sa), + NULL, NULL) < 0)) + goto alg_err; + + state->ip_tot_len -= state->in.trl_len; + + return 0; + +proto_err: + status->error.proto = 1; + + return -1; + +alg_err: + status->error.alg = 1; + + return -1; +} + +static int ipsec_in_strip_tunnel(odp_packet_t *pkt, ipsec_state_t *state, + odp_ipsec_op_status_t *status) +{ + odp_packet_move_data(*pkt, state->ip_hdr_len + state->in.hdr_len, 0, state->ip_offset); + + if (odp_unlikely(odp_packet_trunc_head(pkt, state->ip_hdr_len + state->in.hdr_len, NULL, + NULL) < 0)) { + status->error.alg = 1; + + return -1; + } + + state->ip_tot_len -= state->ip_hdr_len + state->in.hdr_len; + + if (odp_unlikely(!(_ODP_IPPROTO_IPIP == state->ip_next_hdr || + _ODP_IPPROTO_IPV6 == state->ip_next_hdr || + _ODP_IPPROTO_NO_NEXT == state->ip_next_hdr))) { status->error.proto = 1; - goto post_lifetime_err_cnt_update; + + return -1; } - if (odp_packet_trunc_tail(&pkt, state.in.trl_len + ipsec_get_seqh_len(ipsec_sa), - NULL, NULL) < 0) { + state->is_ipv4 = _ODP_IPPROTO_IPIP == state->ip_next_hdr; + state->is_ipv6 = _ODP_IPPROTO_IPV6 == state->ip_next_hdr; + + return 0; +} + +static int ipsec_in_strip_tp(odp_packet_t *pkt, ipsec_state_t *state, + odp_ipsec_op_status_t *status) +{ + odp_packet_move_data(*pkt, state->in.hdr_len, 0, state->ip_offset + state->ip_hdr_len); + + if (odp_unlikely(odp_packet_trunc_head(pkt, state->in.hdr_len, NULL, NULL) < 0)) { status->error.alg = 1; - goto post_lifetime_err_cnt_update; - } - state.ip_tot_len -= state.in.trl_len; - if (ODP_IPSEC_MODE_TUNNEL == ipsec_sa->mode) { - /* We have a tunneled IPv4 packet, strip outer and IPsec - * headers */ - odp_packet_move_data(pkt, state.ip_hdr_len + state.in.hdr_len, - 0, - state.ip_offset); - if (odp_packet_trunc_head(&pkt, state.ip_hdr_len + - state.in.hdr_len, - NULL, NULL) < 0) { - status->error.alg = 1; - goto post_lifetime_err_cnt_update; - } - state.ip_tot_len -= state.ip_hdr_len + state.in.hdr_len; - if (_ODP_IPPROTO_IPIP == state.ip_next_hdr) { - state.is_ipv4 = 1; - state.is_ipv6 = 0; - } else if (_ODP_IPPROTO_IPV6 == state.ip_next_hdr) { - state.is_ipv4 = 0; - state.is_ipv6 = 1; - } else if (_ODP_IPPROTO_NO_NEXT == state.ip_next_hdr) { - state.is_ipv4 = 0; - state.is_ipv6 = 0; - } else { - status->error.proto = 1; - goto post_lifetime_err_cnt_update; - } - } else { - odp_packet_move_data(pkt, state.in.hdr_len, 0, - state.ip_offset + state.ip_hdr_len); - if (odp_packet_trunc_head(&pkt, state.in.hdr_len, - NULL, NULL) < 0) { - status->error.alg = 1; - goto post_lifetime_err_cnt_update; - } - state.ip_tot_len -= state.in.hdr_len; + return -1; } - /* Finalize the IPv4 header */ - if (state.is_ipv4 && odp_packet_len(pkt) > _ODP_IPV4HDR_LEN) { - _odp_ipv4hdr_t *ipv4hdr = odp_packet_l3_ptr(pkt, NULL); + state->ip_tot_len -= state->in.hdr_len; + + return 0; +} + +static int ipsec_in_strip_headers(odp_packet_t *pkt, ipsec_state_t *state, ipsec_sa_t *ipsec_sa, + odp_ipsec_op_status_t *status) +{ + int (*op)(odp_packet_t *pkt, ipsec_state_t *state, odp_ipsec_op_status_t *status); + + op = ODP_IPSEC_MODE_TUNNEL == ipsec_sa->mode ? ipsec_in_strip_tunnel : ipsec_in_strip_tp; + + return op(pkt, state, status); +} + +static int ipsec_in_finalize_decap_header(odp_packet_t pkt, ipsec_state_t *state, + ipsec_sa_t *ipsec_sa, odp_ipsec_op_status_t *status) +{ + _odp_ipv4hdr_t *ipv4hdr; + _odp_ipv6hdr_t *ipv6hdr; + + if (state->is_ipv4 && odp_packet_len(pkt) > _ODP_IPV4HDR_LEN) { + ipv4hdr = odp_packet_l3_ptr(pkt, NULL); if (ODP_IPSEC_MODE_TRANSPORT == ipsec_sa->mode) - ipv4hdr->tot_len = odp_cpu_to_be_16(state.ip_tot_len); + ipv4hdr->tot_len = odp_cpu_to_be_16(state->ip_tot_len); else ipv4hdr->ttl -= ipsec_sa->dec_ttl; + _odp_packet_ipv4_chksum_insert(pkt); - } else if (state.is_ipv6 && odp_packet_len(pkt) > _ODP_IPV6HDR_LEN) { - _odp_ipv6hdr_t *ipv6hdr = odp_packet_l3_ptr(pkt, NULL); + } else if (state->is_ipv6 && odp_packet_len(pkt) > _ODP_IPV6HDR_LEN) { + ipv6hdr = odp_packet_l3_ptr(pkt, NULL); if (ODP_IPSEC_MODE_TRANSPORT == ipsec_sa->mode) - ipv6hdr->payload_len = - odp_cpu_to_be_16(state.ip_tot_len - - _ODP_IPV6HDR_LEN); + ipv6hdr->payload_len = odp_cpu_to_be_16(state->ip_tot_len - + _ODP_IPV6HDR_LEN); else ipv6hdr->hop_limit -= ipsec_sa->dec_ttl; - } else if (state.ip_next_hdr != _ODP_IPPROTO_NO_NEXT) { + } else if (state->ip_next_hdr != _ODP_IPPROTO_NO_NEXT) { status->error.proto = 1; - goto post_lifetime_err_cnt_update; + + return -1; } - if (_ODP_IPPROTO_NO_NEXT == state.ip_next_hdr && - ODP_IPSEC_MODE_TUNNEL == ipsec_sa->mode) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + return 0; +} - packet_parse_reset(pkt_hdr, 0); - pkt_hdr->p.l3_offset = state.ip_offset; - } else { - odp_packet_parse_param_t parse_param; +static int ipsec_in_finalize_packet(odp_packet_t *pkt, ipsec_state_t *state, ipsec_sa_t *ipsec_sa, + odp_ipsec_op_status_t *status) +{ + return _odp_ipsec_sa_lifetime_update(ipsec_sa, state->stats_length, status) < 0 || + ipsec_in_finalize_op(pkt, state, ipsec_sa, status) || + ipsec_in_strip_headers(pkt, state, ipsec_sa, status) || + ipsec_in_finalize_decap_header(*pkt, state, ipsec_sa, status); +} - parse_param.proto = state.is_ipv4 ? ODP_PROTO_IPV4 : - state.is_ipv6 ? ODP_PROTO_IPV6 : - ODP_PROTO_NONE; - parse_param.last_layer = ipsec_config->inbound.parse_level; - parse_param.chksums = ipsec_config->inbound.chksums; +static void ipsec_in_reset_parse_data(odp_packet_t pkt, ipsec_state_t *state) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - /* We do not care about return code here. - * Parsing error should not result in IPsec error. */ - odp_packet_parse(pkt, state.ip_offset, &parse_param); + packet_parse_reset(pkt_hdr, 0); + pkt_hdr->p.l3_offset = state->ip_offset; +} + +static void ipsec_in_parse_packet(odp_packet_t pkt, ipsec_state_t *state) +{ + odp_packet_parse_param_t parse_param; + + parse_param.proto = state->is_ipv4 ? ODP_PROTO_IPV4 : + state->is_ipv6 ? ODP_PROTO_IPV6 : + ODP_PROTO_NONE; + parse_param.last_layer = ipsec_config->inbound.parse_level; + parse_param.chksums = ipsec_config->inbound.chksums; + /* We do not care about return code here. Parsing error should not result in IPsec + * error. */ + odp_packet_parse(pkt, state->ip_offset, &parse_param); +} + +static void ipsec_in_parse_decap_packet(odp_packet_t pkt, ipsec_state_t *state, + ipsec_sa_t *ipsec_sa) +{ + void (*op)(odp_packet_t pkt, ipsec_state_t *state); + + op = _ODP_IPPROTO_NO_NEXT == state->ip_next_hdr && + ODP_IPSEC_MODE_TUNNEL == ipsec_sa->mode ? ipsec_in_reset_parse_data : + ipsec_in_parse_packet; + + op(pkt, state); +} + +static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, + odp_ipsec_sa_t sa, + odp_packet_t *pkt_out, + odp_bool_t enqueue_op, + odp_ipsec_op_status_t *status, + uint32_t *orig_ip_len) +{ + ipsec_state_t state; + ipsec_sa_t *ipsec_sa = NULL; + odp_crypto_packet_op_param_t param; + + if (odp_unlikely(ipsec_in_prepare_packet(&pkt, &state, &ipsec_sa, sa, ¶m, status, + orig_ip_len))) + goto exit; + + if (ipsec_in_do_crypto(&pkt, ¶m, status)) + goto exit; + + if (ipsec_sa->antireplay) { + if (enqueue_op) + wait_for_order(ipsec_global->inbound_ordering_mode); + + if (_odp_ipsec_sa_replay_update(ipsec_sa, state.in.seq_no, status) < 0) + goto exit; } + if (odp_unlikely(ipsec_in_finalize_packet(&pkt, &state, ipsec_sa, status))) + goto post_lifetime_err_cnt_update; + + ipsec_in_parse_decap_packet(pkt, &state, ipsec_sa); + goto exit; post_lifetime_err_cnt_update: if (ipsec_config->stats_en) { odp_atomic_inc_u64(&ipsec_sa->stats.post_lifetime_err_pkts); - odp_atomic_add_u64(&ipsec_sa->stats.post_lifetime_err_bytes, - state.stats_length); + odp_atomic_add_u64(&ipsec_sa->stats.post_lifetime_err_bytes, state.stats_length); } exit: @@ -1101,7 +1208,7 @@ uint64_t ipsec_seq_no(ipsec_sa_t *ipsec_sa) */ static inline uint32_t ipsec_padded_len(uint32_t len, uint32_t pad_mask) { - ODP_ASSERT(_ODP_CHECK_IS_POWER2(pad_mask + 1)); + _ODP_ASSERT(_ODP_CHECK_IS_POWER2(pad_mask + 1)); return (len + pad_mask) & ~pad_mask; } @@ -1295,7 +1402,7 @@ static int ipsec_out_iv(ipsec_state_t *state, { if (ipsec_sa->use_counter_iv) { /* Both GCM and CTR use 8-bit counters */ - ODP_ASSERT(sizeof(seq_no) == ipsec_sa->esp_iv_len); + _ODP_ASSERT(sizeof(seq_no) == ipsec_sa->esp_iv_len); /* Check for overrun */ if (seq_no == 0) @@ -1475,7 +1582,7 @@ static int ipsec_out_esp(odp_packet_t *pkt, if (odp_packet_extend_tail(pkt, IPSEC_SEQ_HI_LEN, NULL, NULL) < 0) { status->error.alg = 1; - ODP_ERR("odp_packet_extend_tail failed\n"); + _ODP_ERR("odp_packet_extend_tail failed\n"); return -1; } odp_packet_copy_from_mem(*pkt, @@ -1506,6 +1613,7 @@ static int ipsec_out_esp(odp_packet_t *pkt, ipsec_sa->icv_len; state->stats_length = param->cipher_range.length; + param->session = ipsec_sa->session; return 0; } @@ -1525,7 +1633,7 @@ static int ipsec_out_esp_post(ipsec_state_t *state, odp_packet_t *pkt, odp_packet_move_data(*pkt, icv_offset - IPSEC_SEQ_HI_LEN, icv_offset, ipsec_sa->icv_len); if (odp_packet_trunc_tail(pkt, IPSEC_SEQ_HI_LEN, NULL, NULL) < 0) { - ODP_ERR("odp_packet_trunc_tail failed\n"); + _ODP_ERR("odp_packet_trunc_tail failed\n"); return -1; } } @@ -1631,7 +1739,7 @@ static int ipsec_out_ah(odp_packet_t *pkt, if (odp_packet_extend_tail(pkt, IPSEC_SEQ_HI_LEN, NULL, NULL) < 0) { status->error.alg = 1; - ODP_ERR("odp_packet_extend_tail failed\n"); + _ODP_ERR("odp_packet_extend_tail failed\n"); return -1; } odp_packet_copy_from_mem(*pkt, @@ -1645,6 +1753,7 @@ static int ipsec_out_ah(odp_packet_t *pkt, state->stats_length = param->auth_range.length; param->auth_range.length += ipsec_get_seqh_len(ipsec_sa); + param->session = ipsec_sa->session; return 0; } @@ -1672,7 +1781,7 @@ static int ipsec_out_ah_post(ipsec_state_t *state, odp_packet_t *pkt, */ if (ipsec_sa->insert_seq_hi) { if (odp_packet_trunc_tail(pkt, IPSEC_SEQ_HI_LEN, NULL, NULL) < 0) { - ODP_ERR("odp_packet_trunc_tail failed\n"); + _ODP_ERR("odp_packet_trunc_tail failed\n"); return -1; } } @@ -1725,425 +1834,467 @@ static void ipsec_out_checksums(odp_packet_t pkt, _odp_packet_sctp_chksum_insert(pkt); } -static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, - odp_ipsec_sa_t sa, - odp_packet_t *pkt_out, - const odp_ipsec_out_opt_t *opt, - odp_bool_t enqueue_op, - odp_ipsec_op_status_t *status) +static int ipsec_out_tp_encap(odp_packet_t pkt, ipsec_state_t *state) { - ipsec_state_t state; - ipsec_sa_t *ipsec_sa; - odp_crypto_packet_op_param_t param; - int rc; - odp_crypto_packet_result_t crypto; /**< Crypto operation result */ - odp_ipsec_frag_mode_t frag_mode; - uint32_t mtu; + int (*op)(ipsec_state_t *state, odp_packet_t pkt); - /* - * No need to do _odp_ipsec_sa_use() here since an ODP application - * is not allowed to do call IPsec output before SA creation has - * completed nor call odp_ipsec_sa_disable() before IPsec output - * has completed. IOW, the needed sychronization between threads - * is done by the application. - */ - ipsec_sa = _odp_ipsec_sa_entry_from_hdl(sa); - ODP_ASSERT(NULL != ipsec_sa); + if (odp_unlikely(!(state->is_ipv4 || state->is_ipv6))) + return -1; - if (opt->flag.tfc_dummy) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + op = state->is_ipv4 ? ipsec_parse_ipv4 : ipsec_parse_ipv6; - ODP_ASSERT(ODP_IPSEC_MODE_TUNNEL == ipsec_sa->mode); - pkt_hdr->p.l2_offset = ODP_PACKET_OFFSET_INVALID; - pkt_hdr->p.l3_offset = 0; - state.ip_offset = 0; - state.ip = NULL; - state.is_ipv4 = 0; - state.is_ipv6 = 0; - } else { - state.ip_offset = odp_packet_l3_offset(pkt); - ODP_ASSERT(ODP_PACKET_OFFSET_INVALID != state.ip_offset); + if (odp_unlikely(op(state, pkt) || + state->ip_tot_len + state->ip_offset != odp_packet_len(pkt))) + return -1; - state.ip = odp_packet_l3_ptr(pkt, NULL); - ODP_ASSERT(NULL != state.ip); + ipsec_out_checksums(pkt, state); - state.is_ipv4 = (((uint8_t *)state.ip)[0] >> 4) == 0x4; - state.is_ipv6 = (((uint8_t *)state.ip)[0] >> 4) == 0x6; - } + return 0; +} - frag_mode = opt->flag.frag_mode ? opt->frag_mode : - ipsec_sa->out.frag_mode; - if (frag_mode == ODP_IPSEC_FRAG_CHECK) - mtu = odp_atomic_load_u32(&ipsec_sa->out.mtu); - else - mtu = UINT32_MAX; +static int ipsec_out_tunnel_encap(odp_packet_t *pkt, ipsec_state_t *state, ipsec_sa_t *ipsec_sa, + const odp_ipsec_out_opt_t *opt) +{ + int ret; - /* Initialize parameters block */ - memset(¶m, 0, sizeof(param)); + if (odp_unlikely(!(state->is_ipv4 || state->is_ipv6 || opt->flag.tfc_dummy))) + return -1; - if (ODP_IPSEC_MODE_TRANSPORT == ipsec_sa->mode) { - if (state.is_ipv4) - rc = ipsec_parse_ipv4(&state, pkt); - else if (state.is_ipv6) - rc = ipsec_parse_ipv6(&state, pkt); - else - rc = -1; - - if (rc == 0) { - if (state.ip_tot_len + state.ip_offset != - odp_packet_len(pkt)) - rc = -1; - else - ipsec_out_checksums(pkt, &state); - } + if (state->is_ipv4) { + if (odp_unlikely(ipsec_out_tunnel_parse_ipv4(state, ipsec_sa))) + return -1; + } else if (state->is_ipv6) { + if (odp_unlikely(ipsec_out_tunnel_parse_ipv6(state, ipsec_sa))) + return -1; } else { - if (state.is_ipv4) { - rc = ipsec_out_tunnel_parse_ipv4(&state, ipsec_sa); - } else if (state.is_ipv6) { - rc = ipsec_out_tunnel_parse_ipv6(&state, ipsec_sa); - } else if (opt->flag.tfc_dummy) { - state.out_tunnel.ip_tos = 0; - state.out_tunnel.ip_df = 0; - state.out_tunnel.ip_flabel = 0; - state.ip_next_hdr = _ODP_IPPROTO_NO_NEXT; - rc = 0; - } else { - rc = -1; - } + state->out_tunnel.ip_tos = 0; + state->out_tunnel.ip_df = 0; + state->out_tunnel.ip_flabel = 0; + state->ip_next_hdr = _ODP_IPPROTO_NO_NEXT; + } - if (rc < 0) { - status->error.alg = 1; - goto exit; - } + ipsec_out_checksums(*pkt, state); - ipsec_out_checksums(pkt, &state); + if (ipsec_sa->tun_ipv4) + ret = ipsec_out_tunnel_ipv4(pkt, state, ipsec_sa, + opt->flag.ip_param ? &opt->ipv4 : + &ipsec_sa->out.tun_ipv4.param); + else + ret = ipsec_out_tunnel_ipv6(pkt, state, ipsec_sa, + opt->flag.ip_param ? &opt->ipv6 : + &ipsec_sa->out.tun_ipv6.param); - if (ipsec_sa->tun_ipv4) - rc = ipsec_out_tunnel_ipv4(&pkt, &state, ipsec_sa, - opt->flag.ip_param ? - &opt->ipv4 : - &ipsec_sa->out.tun_ipv4.param); - else - rc = ipsec_out_tunnel_ipv6(&pkt, &state, ipsec_sa, - opt->flag.ip_param ? - &opt->ipv6 : - &ipsec_sa->out.tun_ipv6.param); - } - if (rc < 0) { - status->error.alg = 1; - goto exit; - } + return ret; +} + +static int ipsec_out_parse_encap_packet(odp_packet_t *pkt, ipsec_state_t *state, + ipsec_sa_t *ipsec_sa, const odp_ipsec_out_opt_t *opt, + odp_ipsec_op_status_t *status) +{ + odp_packet_hdr_t *pkt_hdr; + int ret; - if (ODP_IPSEC_ESP == ipsec_sa->proto) { - rc = ipsec_out_esp(&pkt, &state, ipsec_sa, ¶m, status, mtu, - enqueue_op, opt); - } else if (ODP_IPSEC_AH == ipsec_sa->proto) { - rc = ipsec_out_ah(&pkt, &state, ipsec_sa, ¶m, status, mtu, - enqueue_op); + if (opt->flag.tfc_dummy) { + pkt_hdr = packet_hdr(*pkt); + _ODP_ASSERT(ODP_IPSEC_MODE_TUNNEL == ipsec_sa->mode); + pkt_hdr->p.l2_offset = ODP_PACKET_OFFSET_INVALID; + pkt_hdr->p.l3_offset = 0; + state->ip_offset = 0; + state->ip = NULL; + state->is_ipv4 = 0; + state->is_ipv6 = 0; } else { - status->error.alg = 1; - goto exit; + state->ip_offset = odp_packet_l3_offset(*pkt); + _ODP_ASSERT(ODP_PACKET_OFFSET_INVALID != state->ip_offset); + state->ip = odp_packet_l3_ptr(*pkt, NULL); + _ODP_ASSERT(NULL != state->ip); + state->is_ipv4 = (((uint8_t *)state->ip)[0] >> 4) == 0x4; + state->is_ipv6 = (((uint8_t *)state->ip)[0] >> 4) == 0x6; } - if (rc < 0) - goto exit; - /* No need to run precheck here, we know that packet is authentic */ - if (_odp_ipsec_sa_lifetime_update(ipsec_sa, - state.stats_length, - status) < 0) - goto post_lifetime_err_cnt_update; - - param.session = ipsec_sa->session; + if (ODP_IPSEC_MODE_TRANSPORT == ipsec_sa->mode) + ret = ipsec_out_tp_encap(*pkt, state); + else + ret = ipsec_out_tunnel_encap(pkt, state, ipsec_sa, opt); - /* - * NOTE: Do not change to an asynchronous design without thinking - * concurrency and what changes are required to guarantee that - * used SAs are not destroyed when asynchronous operations are in - * progress. - * - * The containing code does not hold a reference to the SA but - * completes outbound processing synchronously and makes use of - * the fact that the application may not disable (and then destroy) - * the SA before this output routine returns (and all its side - * effects are visible to the disabling thread). - */ - rc = odp_crypto_op(&pkt, &pkt, ¶m, 1); - if (rc < 0) { - ODP_DBG("Crypto failed\n"); + if (odp_unlikely(ret)) status->error.alg = 1; - goto post_lifetime_err_cnt_update; - } - rc = odp_crypto_result(&crypto, pkt); - if (rc < 0) { - ODP_DBG("Crypto failed\n"); - status->error.alg = 1; - goto post_lifetime_err_cnt_update; - } + return ret; +} + +static int ipsec_out_prepare_op(odp_packet_t *pkt, ipsec_state_t *state, ipsec_sa_t *ipsec_sa, + const odp_ipsec_out_opt_t *opt, odp_bool_t is_enqueue_op, + odp_crypto_packet_op_param_t *param, odp_ipsec_op_status_t *status) +{ + odp_ipsec_frag_mode_t frag_mode; + uint32_t mtu; + int ret; - if (!crypto.ok) { + memset(param, 0, sizeof(*param)); + + frag_mode = opt->flag.frag_mode ? opt->frag_mode : ipsec_sa->out.frag_mode; + mtu = frag_mode == ODP_IPSEC_FRAG_CHECK ? odp_atomic_load_u32(&ipsec_sa->out.mtu) : + UINT32_MAX; + + if (odp_unlikely(!(ODP_IPSEC_ESP == ipsec_sa->proto || ODP_IPSEC_AH == ipsec_sa->proto))) { status->error.alg = 1; - goto post_lifetime_err_cnt_update; + + return -1; } - /* Finalize the IP header */ if (ODP_IPSEC_ESP == ipsec_sa->proto) - rc = ipsec_out_esp_post(&state, &pkt, ipsec_sa); - else if (ODP_IPSEC_AH == ipsec_sa->proto) - rc = ipsec_out_ah_post(&state, &pkt, ipsec_sa); + ret = ipsec_out_esp(pkt, state, ipsec_sa, param, status, mtu, is_enqueue_op, opt); + else + ret = ipsec_out_ah(pkt, state, ipsec_sa, param, status, mtu, is_enqueue_op); - if (rc < 0) { - status->error.alg = 1; - goto post_lifetime_err_cnt_update; - } + return ret; +} - goto exit; +static int ipsec_out_prepare_packet(odp_packet_t *pkt, ipsec_state_t *state, ipsec_sa_t *ipsec_sa, + const odp_ipsec_out_opt_t *opt, odp_bool_t is_enqueue_op, + odp_crypto_packet_op_param_t *param, + odp_ipsec_op_status_t *status) +{ + return ipsec_out_parse_encap_packet(pkt, state, ipsec_sa, opt, status) || + ipsec_out_prepare_op(pkt, state, ipsec_sa, opt, is_enqueue_op, param, status); +} -post_lifetime_err_cnt_update: - if (ipsec_config->stats_en) { - odp_atomic_inc_u64(&ipsec_sa->stats.post_lifetime_err_pkts); - odp_atomic_add_u64(&ipsec_sa->stats.post_lifetime_err_bytes, - state.stats_length); +static int ipsec_out_finalize_packet(odp_packet_t *pkt, ipsec_state_t *state, ipsec_sa_t *ipsec_sa, + odp_ipsec_op_status_t *status) +{ + int (*op)(ipsec_state_t *state, odp_packet_t *pkt, ipsec_sa_t *ipsec_sa); + + op = ODP_IPSEC_ESP == ipsec_sa->proto ? ipsec_out_esp_post : + ODP_IPSEC_AH == ipsec_sa->proto ? ipsec_out_ah_post : NULL; + + if (odp_unlikely(op && op(state, pkt, ipsec_sa))) { + status->error.alg = 1; + + return -1; } -exit: - *pkt_out = pkt; - return ipsec_sa; + return 0; } -int odp_ipsec_in(const odp_packet_t pkt_in[], int num_in, - odp_packet_t pkt_out[], int *num_out, - const odp_ipsec_in_param_t *param) +static void ipsec_in_prepare(const odp_packet_t pkt_in[], odp_packet_t pkt_out[], int num_in, + const odp_ipsec_in_param_t *param, ipsec_op_t ops[], + odp_packet_t crypto_pkts[], + odp_crypto_packet_op_param_t crypto_param[], ipsec_op_t *crypto_ops[], + int *num_crypto) { - int in_pkt = 0; - int out_pkt = 0; - int max_out = *num_out; - unsigned sa_idx = 0; - unsigned sa_inc = (param->num_sa > 1) ? 1 : 0; + unsigned int sa_idx = 0, sa_inc = (param->num_sa > 1) ? 1 : 0; + + *num_crypto = 0; - while (in_pkt < num_in && out_pkt < max_out) { - odp_packet_t pkt = pkt_in[in_pkt]; - odp_ipsec_op_status_t status; - odp_ipsec_sa_t sa; - ipsec_sa_t *ipsec_sa; - uint32_t dummy; /* orig_ip_len not valid in sync operations */ - odp_ipsec_packet_result_t *result; + for (int i = 0; i < num_in; i++) { + pkt_out[i] = pkt_in[i]; + ipsec_op_t *op = &ops[i]; + odp_packet_t *pkt = &pkt_out[i]; + odp_crypto_packet_op_param_t c_p; - memset(&status, 0, sizeof(status)); + memset(op, 0, sizeof(*op)); if (0 == param->num_sa) { - sa = ODP_IPSEC_SA_INVALID; + op->sa_hdl = ODP_IPSEC_SA_INVALID; } else { - sa = param->sa[sa_idx]; - ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); + op->sa_hdl = param->sa[sa_idx]; + _ODP_ASSERT(ODP_IPSEC_SA_INVALID != op->sa_hdl); } - ipsec_sa = ipsec_in_single(pkt, sa, &pkt, false, &status, &dummy); - - packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); - result = ipsec_pkt_result(pkt); - memset(result, 0, sizeof(*result)); - result->status = status; - if (NULL != ipsec_sa) - result->sa = ipsec_sa->ipsec_sa_hdl; - else - result->sa = ODP_IPSEC_SA_INVALID; - - pkt_out[out_pkt] = pkt; - in_pkt++; - out_pkt++; sa_idx += sa_inc; - /* - * We need to decrease SA use count only if the SA was not - * provided to us by the caller but was found through our own - * SA lookup that increased the use count. - */ - if (sa == ODP_IPSEC_SA_INVALID && ipsec_sa) - _odp_ipsec_sa_unuse(ipsec_sa); + if (odp_likely(ipsec_in_prepare_packet(pkt, &op->state, &op->sa, op->sa_hdl, &c_p, + &op->status, &op->orig_ip_len) == 0)) { + crypto_pkts[*num_crypto] = *pkt; + crypto_param[*num_crypto] = c_p; + crypto_ops[*num_crypto] = op; + (*num_crypto)++; + } } +} - *num_out = out_pkt; +static void ipsec_do_crypto_burst(odp_packet_t pkts[], odp_crypto_packet_op_param_t param[], + ipsec_op_t *ops[], int num) +{ + int num_procd = 0; - return in_pkt; -} + while (num_procd < num) { + int ret = odp_crypto_op(&pkts[num_procd], &pkts[num_procd], ¶m[num_procd], + num - num_procd); -static odp_ipsec_out_opt_t default_out_opt; + if (odp_unlikely(ret <= 0)) + break; -int odp_ipsec_out(const odp_packet_t pkt_in[], int num_in, - odp_packet_t pkt_out[], int *num_out, - const odp_ipsec_out_param_t *param) -{ - int in_pkt = 0; - int out_pkt = 0; - int max_out = *num_out; - unsigned sa_idx = 0; - unsigned opt_idx = 0; - unsigned sa_inc = (param->num_sa > 1) ? 1 : 0; - unsigned opt_inc = (param->num_opt > 1) ? 1 : 0; - - ODP_ASSERT(param->num_sa != 0); - - while (in_pkt < num_in && out_pkt < max_out) { - odp_packet_t pkt = pkt_in[in_pkt]; - odp_ipsec_op_status_t status; - odp_ipsec_sa_t sa; - ipsec_sa_t *ipsec_sa; - odp_ipsec_packet_result_t *result; - const odp_ipsec_out_opt_t *opt; + num_procd += ret; + } - memset(&status, 0, sizeof(status)); + for (int i = num_procd; i < num; i++) + ops[i]->status.error.alg = 1; +} - sa = param->sa[sa_idx]; - ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); +static int ipsec_in_check_crypto_result(odp_packet_t pkt, odp_ipsec_op_status_t *status) +{ + odp_crypto_packet_result_t crypto; - if (0 == param->num_opt) - opt = &default_out_opt; - else - opt = ¶m->opt[opt_idx]; + if (odp_unlikely(odp_crypto_result(&crypto, pkt) < 0)) { + _ODP_DBG("Crypto failed\n"); + status->error.alg = 1; + return -1; + } - ipsec_sa = ipsec_out_single(pkt, sa, &pkt, opt, false, &status); - ODP_ASSERT(NULL != ipsec_sa); + if (odp_unlikely(!crypto.ok)) { + if (crypto.cipher_status.alg_err == ODP_CRYPTO_ALG_ERR_ICV_CHECK || + crypto.auth_status.alg_err == ODP_CRYPTO_ALG_ERR_ICV_CHECK) + status->error.auth = 1; + else + status->error.alg = 1; - packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); - result = ipsec_pkt_result(pkt); - memset(result, 0, sizeof(*result)); - result->status = status; - result->sa = ipsec_sa->ipsec_sa_hdl; + return -1; + } - if (ipsec_config->stats_en) - ipsec_sa_err_stats_update(ipsec_sa, &status); + return 0; +} - pkt_out[out_pkt] = pkt; - in_pkt++; - out_pkt++; - sa_idx += sa_inc; - opt_idx += opt_inc; +static inline void update_post_lifetime_stats(ipsec_sa_t *sa, ipsec_state_t *state) +{ + if (ipsec_config->stats_en) { + odp_atomic_inc_u64(&sa->stats.post_lifetime_err_pkts); + odp_atomic_add_u64(&sa->stats.post_lifetime_err_bytes, state->stats_length); } +} - *num_out = out_pkt; +static inline void finish_packet_proc(odp_packet_t pkt, ipsec_op_t *op, odp_queue_t queue) +{ + odp_ipsec_packet_result_t *res; - return in_pkt; + if (ipsec_config->stats_en) + ipsec_sa_err_stats_update(op->sa, &op->status); + + packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); + res = ipsec_pkt_result(pkt); + memset(res, 0, sizeof(*res)); + res->status = op->status; + res->sa = NULL != op->sa ? op->sa->ipsec_sa_hdl : ODP_IPSEC_SA_INVALID; + /* We need to decrease SA use count only if the SA was not provided to us by the caller but + * was found through our own SA lookup that increased the use count. */ + if (op->sa_hdl == ODP_IPSEC_SA_INVALID && op->sa) + _odp_ipsec_sa_unuse(op->sa); + + if (queue != ODP_QUEUE_INVALID) { + res->orig_ip_len = op->orig_ip_len; + /* What should be done if enqueue fails? */ + if (odp_unlikely(odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt)) < 0)) + odp_packet_free(pkt); + } } -int odp_ipsec_in_enq(const odp_packet_t pkt_in[], int num_in, - const odp_ipsec_in_param_t *param) +static void ipsec_in_finalize(odp_packet_t pkt_in[], ipsec_op_t ops[], int num, odp_bool_t is_enq) { - int in_pkt = 0; - unsigned sa_idx = 0; - unsigned sa_inc = (param->num_sa > 1) ? 1 : 0; + for (int i = 0; i < num; i++) { + ipsec_op_t *op = &ops[i]; + odp_packet_t *pkt = &pkt_in[i]; + odp_queue_t q = ODP_QUEUE_INVALID; - while (in_pkt < num_in) { - odp_packet_t pkt = pkt_in[in_pkt]; - odp_ipsec_op_status_t status; - odp_ipsec_sa_t sa; - ipsec_sa_t *ipsec_sa; - uint32_t orig_ip_len = 0; - odp_ipsec_packet_result_t *result; - odp_queue_t queue; - int rc; + if (odp_unlikely(op->status.error.all)) + goto finish; - memset(&status, 0, sizeof(status)); + if (odp_unlikely(ipsec_in_check_crypto_result(*pkt, &op->status))) + goto finish; - if (0 == param->num_sa) { - sa = ODP_IPSEC_SA_INVALID; - } else { - sa = param->sa[sa_idx]; - ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); - } + if (op->sa->antireplay) { + if (is_enq) + wait_for_order(ipsec_global->inbound_ordering_mode); - ipsec_sa = ipsec_in_single(pkt, sa, &pkt, true, &status, &orig_ip_len); + if (odp_unlikely(_odp_ipsec_sa_replay_update(op->sa, op->state.in.seq_no, + &op->status) < 0)) + goto finish; + } - packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); - result = ipsec_pkt_result(pkt); - memset(result, 0, sizeof(*result)); - result->status = status; - result->orig_ip_len = orig_ip_len; - if (NULL != ipsec_sa) { - result->sa = ipsec_sa->ipsec_sa_hdl; - queue = ipsec_sa->queue; - } else { - result->sa = ODP_IPSEC_SA_INVALID; - queue = ipsec_config->inbound.default_queue; + if (odp_unlikely(ipsec_in_finalize_packet(pkt, &op->state, op->sa, + &op->status))) { + update_post_lifetime_stats(op->sa, &op->state); + goto finish; } - in_pkt++; - sa_idx += sa_inc; + ipsec_in_parse_decap_packet(*pkt, &op->state, op->sa); - /* - * We need to decrease SA use count only if the SA was not - * provided to us by the caller but was found through our own - * SA lookup that increased the use count. - */ - if (sa == ODP_IPSEC_SA_INVALID && ipsec_sa) - _odp_ipsec_sa_unuse(ipsec_sa); +finish: + if (is_enq) + q = NULL != op->sa ? op->sa->queue : ipsec_config->inbound.default_queue; - rc = odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt)); - if (odp_unlikely(rc)) { - odp_packet_free(pkt); - break; - } + finish_packet_proc(*pkt, op, q); } - - return in_pkt; } -int odp_ipsec_out_enq(const odp_packet_t pkt_in[], int num_in, - const odp_ipsec_out_param_t *param) +int odp_ipsec_in(const odp_packet_t pkt_in[], int num_in, odp_packet_t pkt_out[], int *num_out, + const odp_ipsec_in_param_t *param) { - int in_pkt = 0; - unsigned sa_idx = 0; - unsigned opt_idx = 0; - unsigned sa_inc = (param->num_sa > 1) ? 1 : 0; - unsigned opt_inc = (param->num_opt > 1) ? 1 : 0; + int max_out = _ODP_MIN(_ODP_MIN(num_in, *num_out), MAX_BURST), num_crypto; + odp_packet_t crypto_pkts[MAX_BURST]; + odp_crypto_packet_op_param_t crypto_param[MAX_BURST]; + ipsec_op_t ops[MAX_BURST], *crypto_ops[MAX_BURST]; + + ipsec_in_prepare(pkt_in, pkt_out, max_out, param, ops, crypto_pkts, crypto_param, + crypto_ops, &num_crypto); + ipsec_do_crypto_burst(crypto_pkts, crypto_param, crypto_ops, num_crypto); + ipsec_in_finalize(pkt_out, ops, max_out, false); + *num_out = max_out; + + return max_out; +} - ODP_ASSERT(param->num_sa != 0); +static odp_ipsec_out_opt_t default_out_opt; - while (in_pkt < num_in) { - odp_packet_t pkt = pkt_in[in_pkt]; - odp_ipsec_op_status_t status; - odp_ipsec_sa_t sa; - ipsec_sa_t *ipsec_sa; - odp_ipsec_packet_result_t *result; +static void ipsec_out_prepare(const odp_packet_t pkt_in[], odp_packet_t pkt_out[], int num_in, + const odp_ipsec_out_param_t *param, ipsec_op_t ops[], + odp_packet_t crypto_pkts[], + odp_crypto_packet_op_param_t crypto_param[], + ipsec_op_t *crypto_ops[], int *num_crypto, odp_bool_t is_enq) +{ + unsigned int sa_idx = 0, opt_idx = 0, sa_inc = (param->num_sa > 1) ? 1 : 0, + opt_inc = (param->num_opt > 1) ? 1 : 0; + /* No need to do _odp_ipsec_sa_use() here since an ODP application is not allowed to do + * call IPsec output before SA creation has completed nor call odp_ipsec_sa_disable() + * before IPsec output has completed. IOW, the needed sychronization between threads is + * done by the application. */ + *num_crypto = 0; + + for (int i = 0; i < num_in; i++) { + pkt_out[i] = pkt_in[i]; + ipsec_op_t *op = &ops[i]; const odp_ipsec_out_opt_t *opt; - odp_queue_t queue; - int rc; + odp_packet_t *pkt = &pkt_out[i]; + odp_crypto_packet_op_param_t c_p; - memset(&status, 0, sizeof(status)); - - sa = param->sa[sa_idx]; - ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); + memset(op, 0, sizeof(*op)); + op->sa_hdl = param->sa[sa_idx]; + _ODP_ASSERT(ODP_IPSEC_SA_INVALID != op->sa_hdl); + op->sa = _odp_ipsec_sa_entry_from_hdl(op->sa_hdl); + _ODP_ASSERT(NULL != op->sa); if (0 == param->num_opt) opt = &default_out_opt; else opt = ¶m->opt[opt_idx]; - ipsec_sa = ipsec_out_single(pkt, sa, &pkt, opt, true, &status); - ODP_ASSERT(NULL != ipsec_sa); + sa_idx += sa_inc; + opt_idx += opt_inc; - packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); - result = ipsec_pkt_result(pkt); - memset(result, 0, sizeof(*result)); - result->status = status; - result->sa = ipsec_sa->ipsec_sa_hdl; - queue = ipsec_sa->queue; + if (odp_unlikely(ipsec_out_prepare_packet(pkt, &op->state, op->sa, opt, is_enq, + &c_p, &op->status))) + continue; - if (ipsec_config->stats_en) - ipsec_sa_err_stats_update(ipsec_sa, &status); + if (odp_unlikely(_odp_ipsec_sa_lifetime_update(op->sa, op->state.stats_length, + &op->status))) { + update_post_lifetime_stats(op->sa, &op->state); + continue; + } - in_pkt++; - sa_idx += sa_inc; - opt_idx += opt_inc; + crypto_pkts[*num_crypto] = *pkt; + crypto_param[*num_crypto] = c_p; + crypto_ops[*num_crypto] = op; + (*num_crypto)++; + } +} - rc = odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt)); - if (odp_unlikely(rc)) { - odp_packet_free(pkt); - break; +static int ipsec_out_check_crypto_result(odp_packet_t pkt, odp_ipsec_op_status_t *status) +{ + odp_crypto_packet_result_t crypto; + + if (odp_unlikely(odp_crypto_result(&crypto, pkt) < 0)) { + _ODP_DBG("Crypto failed\n"); + status->error.alg = 1; + return -1; + } + + if (odp_unlikely(!crypto.ok)) { + status->error.alg = 1; + + return -1; + } + + return 0; +} + +static void ipsec_out_finalize(odp_packet_t pkt_in[], ipsec_op_t ops[], int num, odp_bool_t is_enq) +{ + for (int i = 0; i < num; i++) { + ipsec_op_t *op = &ops[i]; + odp_packet_t *pkt = &pkt_in[i]; + odp_queue_t q = ODP_QUEUE_INVALID; + + if (odp_unlikely(op->status.error.all)) + goto finish; + + if (odp_unlikely(ipsec_out_check_crypto_result(*pkt, &op->status))) { + update_post_lifetime_stats(op->sa, &op->state); + goto finish; } + + if (odp_unlikely(ipsec_out_finalize_packet(pkt, &op->state, op->sa, &op->status))) + update_post_lifetime_stats(op->sa, &op->state); + +finish: + if (is_enq) + q = NULL != op->sa ? op->sa->queue : ipsec_config->inbound.default_queue; + + finish_packet_proc(*pkt, op, q); } +} - return in_pkt; +int odp_ipsec_out(const odp_packet_t pkt_in[], int num_in, odp_packet_t pkt_out[], int *num_out, + const odp_ipsec_out_param_t *param) +{ + int max_out = _ODP_MIN(_ODP_MIN(num_in, *num_out), MAX_BURST), num_crypto; + odp_packet_t crypto_pkts[MAX_BURST]; + odp_crypto_packet_op_param_t crypto_param[MAX_BURST]; + ipsec_op_t ops[MAX_BURST], *crypto_ops[MAX_BURST]; + + ipsec_out_prepare(pkt_in, pkt_out, max_out, param, ops, crypto_pkts, crypto_param, + crypto_ops, &num_crypto, false); + ipsec_do_crypto_burst(crypto_pkts, crypto_param, crypto_ops, num_crypto); + ipsec_out_finalize(pkt_out, ops, max_out, false); + *num_out = max_out; + + return max_out; +} + +/* Do not change to an asynchronous design without thinking concurrency and what changes are + * required to guarantee that used SAs are not destroyed when asynchronous operations are in + * progress. + * + * The containing code does not hold a reference to the SA but completes processing synchronously + * and makes use of the fact that the application may not disable (and then destroy) the SA before + * these routines return (and all side effects are visible to the disabling thread). */ +int odp_ipsec_in_enq(const odp_packet_t pkt_in[], int num_in, const odp_ipsec_in_param_t *param) +{ + int max_out = _ODP_MIN(num_in, MAX_BURST), num_crypto; + odp_packet_t pkt_out[MAX_BURST], crypto_pkts[MAX_BURST]; + odp_crypto_packet_op_param_t crypto_param[MAX_BURST]; + ipsec_op_t ops[MAX_BURST], *crypto_ops[MAX_BURST]; + + ipsec_in_prepare(pkt_in, pkt_out, max_out, param, ops, crypto_pkts, crypto_param, + crypto_ops, &num_crypto); + ipsec_do_crypto_burst(crypto_pkts, crypto_param, crypto_ops, num_crypto); + ipsec_in_finalize(pkt_out, ops, max_out, true); + + return max_out; +} + +int odp_ipsec_out_enq(const odp_packet_t pkt_in[], int num_in, const odp_ipsec_out_param_t *param) +{ + int max_out = _ODP_MIN(num_in, MAX_BURST), num_crypto; + odp_packet_t pkt_out[MAX_BURST], crypto_pkts[MAX_BURST]; + odp_crypto_packet_op_param_t crypto_param[MAX_BURST]; + ipsec_op_t ops[MAX_BURST], *crypto_ops[MAX_BURST]; + + ipsec_out_prepare(pkt_in, pkt_out, max_out, param, ops, crypto_pkts, crypto_param, + crypto_ops, &num_crypto, true); + ipsec_do_crypto_burst(crypto_pkts, crypto_param, crypto_ops, num_crypto); + ipsec_out_finalize(pkt_out, ops, max_out, true); + + return max_out; } int _odp_ipsec_try_inline(odp_packet_t *pkt) @@ -2188,130 +2339,180 @@ int _odp_ipsec_try_inline(odp_packet_t *pkt) return 0; } -#define MAX_HDR_LEN 100 /* Enough for VxLAN over IPv6 */ - -int odp_ipsec_out_inline(const odp_packet_t pkt_in[], int num_in, - const odp_ipsec_out_param_t *param, - const odp_ipsec_out_inline_param_t *inline_param) +static inline int ipsec_out_inline_check_out_hdrs(odp_packet_t pkt, + const odp_ipsec_out_inline_param_t *param, + ipsec_inline_op_t *op) { - int in_pkt = 0; - unsigned sa_idx = 0; - unsigned opt_idx = 0; - unsigned sa_inc = (param->num_sa > 1) ? 1 : 0; - unsigned opt_inc = (param->num_opt > 1) ? 1 : 0; - uint8_t hdr_buf[MAX_HDR_LEN]; + uint32_t l2_offset, hdr_len = param->outer_hdr.len; - ODP_ASSERT(param->num_sa != 0); + if (!param->outer_hdr.ptr) { + l2_offset = odp_packet_l2_offset(pkt); + _ODP_ASSERT(hdr_len == odp_packet_l3_offset(pkt) - l2_offset); - while (in_pkt < num_in) { - odp_packet_t pkt = pkt_in[in_pkt]; - odp_ipsec_op_status_t status; - odp_ipsec_sa_t sa; - ipsec_sa_t *ipsec_sa; - odp_ipsec_packet_result_t *result; - const odp_ipsec_out_opt_t *opt; - uint32_t hdr_len, offset; - const void *ptr; + if (odp_unlikely(hdr_len > MAX_HDR_LEN || + odp_packet_copy_to_mem(pkt, l2_offset, hdr_len, op->hdr_buf) + < 0)) { + op->op.status.error.proto = 1; - ODP_ASSERT(inline_param[in_pkt].pktio != ODP_PKTIO_INVALID); - - memset(&status, 0, sizeof(status)); - - if (0 == param->num_sa) { - sa = ODP_IPSEC_SA_INVALID; - } else { - sa = param->sa[sa_idx]; - ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); + return -1; } + } + + return 0; +} - hdr_len = inline_param[in_pkt].outer_hdr.len; - ptr = inline_param[in_pkt].outer_hdr.ptr; +static void ipsec_out_inline_prepare(const odp_packet_t pkt_in[], odp_packet_t pkt_out[], + int num_in, const odp_ipsec_out_param_t *param, + const odp_ipsec_out_inline_param_t *inline_param, + ipsec_inline_op_t ops[], odp_packet_t crypto_pkts[], + odp_crypto_packet_op_param_t crypto_param[], + ipsec_op_t *crypto_ops[], int *num_crypto) +{ + unsigned int sa_idx = 0, opt_idx = 0, sa_inc = (param->num_sa > 1) ? 1 : 0, + opt_inc = (param->num_opt > 1) ? 1 : 0; - if (!ptr) { - uint32_t l2_offset = odp_packet_l2_offset(pkt); + *num_crypto = 0; - ODP_ASSERT(hdr_len == odp_packet_l3_offset(pkt) - l2_offset); + for (int i = 0; i < num_in; i++) { + pkt_out[i] = pkt_in[i]; + ipsec_inline_op_t *op = &ops[i]; + const odp_ipsec_out_opt_t *opt; + odp_packet_t *pkt = &pkt_out[i]; + odp_crypto_packet_op_param_t c_p; - if (odp_likely(hdr_len <= MAX_HDR_LEN) && - odp_likely(odp_packet_copy_to_mem(pkt, l2_offset, - hdr_len, hdr_buf) == 0)) { - ptr = hdr_buf; - } else { - status.error.proto = 1; - ipsec_sa = _odp_ipsec_sa_entry_from_hdl(sa); - goto err; - } - } + memset(op, 0, sizeof(*op)); + op->op.sa_hdl = param->sa[sa_idx]; + _ODP_ASSERT(ODP_IPSEC_SA_INVALID != op->op.sa_hdl); + op->op.sa = _odp_ipsec_sa_entry_from_hdl(op->op.sa_hdl); + _ODP_ASSERT(NULL != op->op.sa); if (0 == param->num_opt) opt = &default_out_opt; else opt = ¶m->opt[opt_idx]; - ipsec_sa = ipsec_out_single(pkt, sa, &pkt, opt, true, &status); - ODP_ASSERT(NULL != ipsec_sa); + sa_idx += sa_inc; + opt_idx += opt_inc; - offset = odp_packet_l3_offset(pkt); - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) - offset = 0; - if (offset >= hdr_len) { - if (odp_packet_trunc_head(&pkt, offset - hdr_len, - NULL, NULL) < 0) - status.error.alg = 1; + if (odp_unlikely(ipsec_out_inline_check_out_hdrs(*pkt, &inline_param[i], op) || + ipsec_out_prepare_packet(pkt, &op->op.state, op->op.sa, opt, + true, &c_p, &op->op.status))) + continue; - } else { - if (odp_packet_extend_head(&pkt, hdr_len - offset, - NULL, NULL) < 0) - status.error.alg = 1; + if (odp_unlikely(_odp_ipsec_sa_lifetime_update(op->op.sa, + op->op.state.stats_length, + &op->op.status))) { + update_post_lifetime_stats(op->op.sa, &op->op.state); + continue; } - odp_packet_l3_offset_set(pkt, hdr_len); + crypto_pkts[*num_crypto] = *pkt; + crypto_param[*num_crypto] = c_p; + crypto_ops[*num_crypto] = &op->op; + (*num_crypto)++; + } +} - if (odp_packet_copy_from_mem(pkt, 0, - hdr_len, - ptr) < 0) - status.error.alg = 1; +static void ipsec_out_inline_finish_packet_proc(odp_packet_t *pkt, + const odp_ipsec_out_inline_param_t *param, + ipsec_inline_op_t *op) +{ + uint32_t offset = odp_packet_l3_offset(*pkt), hdr_len = param->outer_hdr.len; + odp_pktout_queue_t pkqueue; - if (!status.error.all) { - odp_pktout_queue_t pkqueue; + _ODP_ASSERT(NULL != op->op.sa); - if (odp_pktout_queue(inline_param[in_pkt].pktio, - &pkqueue, 1) <= 0) { - status.error.alg = 1; - goto err; - } + if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) + offset = 0; - if (odp_pktout_send(pkqueue, &pkt, 1) < 0) { - status.error.alg = 1; - goto err; - } - } else { - odp_queue_t queue; - int rc; -err: - if (ipsec_config->stats_en) - ipsec_sa_err_stats_update(ipsec_sa, &status); - - packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); - result = ipsec_pkt_result(pkt); - memset(result, 0, sizeof(*result)); - result->sa = ipsec_sa->ipsec_sa_hdl; - result->status = status; - queue = ipsec_sa->queue; - - rc = odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt)); - if (odp_unlikely(rc)) { - in_pkt++; - odp_packet_free(pkt); - break; - } + if (offset >= hdr_len) { + if (odp_packet_trunc_head(pkt, offset - hdr_len, NULL, NULL) < 0) + op->op.status.error.alg = 1; + } else { + if (odp_packet_extend_head(pkt, hdr_len - offset, NULL, NULL) < 0) + op->op.status.error.alg = 1; + } + + odp_packet_l3_offset_set(*pkt, hdr_len); + + if (odp_packet_copy_from_mem(*pkt, 0, hdr_len, + param->outer_hdr.ptr ? param->outer_hdr.ptr : op->hdr_buf) + < 0) + op->op.status.error.alg = 1; + + if (!op->op.status.error.all) { + if (odp_pktout_queue(param->pktio, &pkqueue, 1) <= 0) + op->op.status.error.alg = 1; + + if (odp_pktout_send(pkqueue, pkt, 1) < 0) + op->op.status.error.alg = 1; + } +} + +static void ipsec_out_inline_handle_err(odp_packet_t pkt, ipsec_inline_op_t *op) +{ + odp_ipsec_packet_result_t *res; + + if (odp_likely(!op->op.status.error.all)) + return; + + if (ipsec_config->stats_en) + ipsec_sa_err_stats_update(op->op.sa, &op->op.status); + + packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); + res = ipsec_pkt_result(pkt); + memset(res, 0, sizeof(*res)); + res->sa = op->op.sa_hdl; + res->status = op->op.status; + + if (odp_unlikely(odp_queue_enq(op->op.sa->queue, odp_ipsec_packet_to_event(pkt)) < 0)) + odp_packet_free(pkt); +} + +static void ipsec_out_inline_finalize(odp_packet_t pkt_in[], + const odp_ipsec_out_inline_param_t *inline_param, + ipsec_inline_op_t ops[], int num) +{ + for (int i = 0; i < num; i++) { + ipsec_inline_op_t *op = &ops[i]; + odp_packet_t *pkt = &pkt_in[i]; + + if (odp_unlikely(op->op.status.error.all)) + goto handle_err; + + if (odp_unlikely(ipsec_out_check_crypto_result(*pkt, &op->op.status))) { + update_post_lifetime_stats(op->op.sa, &op->op.state); + goto finish; } - in_pkt++; - sa_idx += sa_inc; - opt_idx += opt_inc; + + if (odp_unlikely(ipsec_out_finalize_packet(pkt, &op->op.state, op->op.sa, + &op->op.status))) + update_post_lifetime_stats(op->op.sa, &op->op.state); + +finish: + ipsec_out_inline_finish_packet_proc(pkt, &inline_param[i], op); + +handle_err: + ipsec_out_inline_handle_err(*pkt, op); } +} - return in_pkt; +int odp_ipsec_out_inline(const odp_packet_t pkt_in[], int num_in, + const odp_ipsec_out_param_t *param, + const odp_ipsec_out_inline_param_t *inline_param) +{ + int max_out = _ODP_MIN(num_in, MAX_BURST), num_crypto; + odp_packet_t pkt_out[MAX_BURST], crypto_pkts[MAX_BURST]; + odp_crypto_packet_op_param_t crypto_param[MAX_BURST]; + ipsec_inline_op_t ops[MAX_BURST]; + ipsec_op_t *crypto_ops[MAX_BURST]; + + ipsec_out_inline_prepare(pkt_in, pkt_out, max_out, param, inline_param, ops, crypto_pkts, + crypto_param, crypto_ops, &num_crypto); + ipsec_do_crypto_burst(crypto_pkts, crypto_param, crypto_ops, num_crypto); + ipsec_out_inline_finalize(pkt_out, inline_param, ops, max_out); + + return max_out; } int odp_ipsec_test_sa_update(odp_ipsec_sa_t sa, @@ -2321,7 +2522,7 @@ int odp_ipsec_test_sa_update(odp_ipsec_sa_t sa, ipsec_sa_t *ipsec_sa; ipsec_sa = _odp_ipsec_sa_entry_from_hdl(sa); - ODP_ASSERT(NULL != ipsec_sa); + _ODP_ASSERT(NULL != ipsec_sa); switch (sa_op) { case ODP_IPSEC_TEST_SA_UPDATE_SEQ_NUM: @@ -2338,7 +2539,7 @@ int odp_ipsec_result(odp_ipsec_packet_result_t *result, odp_packet_t packet) { odp_ipsec_packet_result_t *res; - ODP_ASSERT(result != NULL); + _ODP_ASSERT(result != NULL); res = ipsec_pkt_result(packet); @@ -2352,8 +2553,8 @@ int odp_ipsec_result(odp_ipsec_packet_result_t *result, odp_packet_t packet) odp_packet_t odp_ipsec_packet_from_event(odp_event_t ev) { - ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); - ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_IPSEC); + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); + _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_IPSEC); return odp_packet_from_event(ev); } @@ -2372,10 +2573,10 @@ int odp_ipsec_stats(odp_ipsec_sa_t sa, odp_ipsec_stats_t *stats) if (!ipsec_config->stats_en) return -ENOTSUP; - ODP_ASSERT(NULL != stats); + _ODP_ASSERT(NULL != stats); ipsec_sa = _odp_ipsec_sa_entry_from_hdl(sa); - ODP_ASSERT(NULL != ipsec_sa); + _ODP_ASSERT(NULL != ipsec_sa); _odp_ipsec_sa_stats_pkts(ipsec_sa, stats); stats->proto_err = odp_atomic_load_u64(&ipsec_sa->stats.proto_err); @@ -2393,7 +2594,7 @@ int odp_ipsec_stats_multi(odp_ipsec_sa_t sa[], odp_ipsec_stats_t stats[], int nu { int ret, i; - ODP_ASSERT(NULL != stats); + _ODP_ASSERT(NULL != stats); for (i = 0; i < num; i++) { ret = odp_ipsec_stats(sa[i], &stats[i]); @@ -2411,13 +2612,13 @@ static int read_config_file(ipsec_global_t *global) int val; if (!_odp_libconfig_lookup_int(str_i, &val)) { - ODP_ERR("Config option '%s' not found.\n", str_i); + _ODP_ERR("Config option '%s' not found.\n", str_i); return -1; } global->inbound_ordering_mode = val; if (!_odp_libconfig_lookup_int(str_o, &val)) { - ODP_ERR("Config option '%s' not found.\n", str_o); + _ODP_ERR("Config option '%s' not found.\n", str_o); return -1; } global->outbound_ordering_mode = val; @@ -2435,12 +2636,12 @@ int _odp_ipsec_init_global(void) shm = odp_shm_reserve("_odp_ipsec_global", sizeof(*ipsec_global), ODP_CACHE_LINE_SIZE, 0); if (shm == ODP_SHM_INVALID) { - ODP_ERR("Shm reserve failed for odp_ipsec\n"); + _ODP_ERR("Shm reserve failed for odp_ipsec\n"); return -1; } ipsec_global = odp_shm_addr(shm); if (ipsec_global == NULL) { - ODP_ERR("ipsec: odp_shm_addr() failed\n"); + _ODP_ERR("ipsec: odp_shm_addr() failed\n"); odp_shm_free(shm); return -1; } @@ -2467,7 +2668,7 @@ int _odp_ipsec_term_global(void) shm = odp_shm_lookup("_odp_ipsec_global"); if (shm == ODP_SHM_INVALID || odp_shm_free(shm)) { - ODP_ERR("Shm free failed for odp_ipsec"); + _ODP_ERR("Shm free failed for odp_ipsec"); return -1; } @@ -2476,16 +2677,16 @@ int _odp_ipsec_term_global(void) void odp_ipsec_print(void) { - ODP_PRINT("\nIPSEC print\n"); - ODP_PRINT("-----------\n"); - ODP_PRINT(" max number of SA %u\n\n", ipsec_config->max_num_sa); + _ODP_PRINT("\nIPSEC print\n"); + _ODP_PRINT("-----------\n"); + _ODP_PRINT(" max number of SA %u\n\n", ipsec_config->max_num_sa); } void odp_ipsec_sa_print(odp_ipsec_sa_t sa) { ipsec_sa_t *ipsec_sa = _odp_ipsec_sa_entry_from_hdl(sa); - ODP_PRINT("\nIPSEC SA print\n"); - ODP_PRINT("--------------\n"); - ODP_PRINT(" SPI %u\n\n", ipsec_sa->spi); + _ODP_PRINT("\nIPSEC SA print\n"); + _ODP_PRINT("--------------\n"); + _ODP_PRINT(" SPI %u\n\n", ipsec_sa->spi); } diff --git a/platform/linux-generic/odp_ipsec_events.c b/platform/linux-generic/odp_ipsec_events.c index f229a5cf2..769cab78b 100644 --- a/platform/linux-generic/odp_ipsec_events.c +++ b/platform/linux-generic/odp_ipsec_events.c @@ -37,7 +37,7 @@ int _odp_ipsec_events_init_global(void) odp_pool_param_t param; if (odp_global_ro.disable.ipsec) { - ODP_PRINT("\nODP IPSec is DISABLED\n"); + _ODP_PRINT("\nODP IPSec is DISABLED\n"); return 0; } @@ -50,7 +50,7 @@ int _odp_ipsec_events_init_global(void) ipsec_status_pool = odp_pool_create("_odp_ipsec_status_pool", ¶m); if (ODP_POOL_INVALID == ipsec_status_pool) { - ODP_ERR("Error: status pool create failed.\n"); + _ODP_ERR("Error: status pool create failed.\n"); goto err_status; } @@ -69,7 +69,7 @@ int _odp_ipsec_events_term_global(void) ret = odp_pool_destroy(ipsec_status_pool); if (ret < 0) { - ODP_ERR("status pool destroy failed"); + _ODP_ERR("status pool destroy failed"); return -1; } @@ -78,15 +78,15 @@ int _odp_ipsec_events_term_global(void) ipsec_status_t _odp_ipsec_status_from_event(odp_event_t ev) { - ODP_ASSERT(ODP_EVENT_INVALID != ev); - ODP_ASSERT(ODP_EVENT_IPSEC_STATUS == odp_event_type(ev)); + _ODP_ASSERT(ODP_EVENT_INVALID != ev); + _ODP_ASSERT(ODP_EVENT_IPSEC_STATUS == odp_event_type(ev)); return (ipsec_status_t)ev; } static odp_event_t ipsec_status_to_event(ipsec_status_t status) { - ODP_ASSERT(ODP_IPSEC_STATUS_INVALID != status); + _ODP_ASSERT(ODP_IPSEC_STATUS_INVALID != status); return (odp_event_t)status; } @@ -96,9 +96,14 @@ static ipsec_status_hdr_t *ipsec_status_hdr_from_buf(odp_buffer_t buf) return (ipsec_status_hdr_t *)(void *)_odp_buf_hdr(buf); } +static inline odp_buffer_t buffer_from_event(odp_event_t ev) +{ + return (odp_buffer_t)ev; +} + static ipsec_status_hdr_t *ipsec_status_hdr(ipsec_status_t status) { - odp_buffer_t buf = odp_buffer_from_event(ipsec_status_to_event(status)); + odp_buffer_t buf = buffer_from_event(ipsec_status_to_event(status)); return ipsec_status_hdr_from_buf(buf); } @@ -119,7 +124,8 @@ void _odp_ipsec_status_free(ipsec_status_t status) { odp_event_t ev = ipsec_status_to_event(status); - odp_buffer_free(odp_buffer_from_event(ev)); + _odp_event_type_set(ev, ODP_EVENT_BUFFER); + odp_buffer_free(buffer_from_event(ev)); } int _odp_ipsec_status_send(odp_queue_t queue, diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index e689089d3..3097bef4b 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -133,7 +133,7 @@ static inline odp_ipsec_sa_t ipsec_sa_index_to_handle(uint32_t ipsec_sa_idx) ipsec_sa_t *_odp_ipsec_sa_entry_from_hdl(odp_ipsec_sa_t sa) { - ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); + _ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); return ipsec_sa_entry_from_hdl(sa); } @@ -171,7 +171,7 @@ int _odp_ipsec_sad_init_global(void) crypto_capa.max_sessions = 0; if (odp_crypto_capability(&crypto_capa)) { - ODP_ERR("odp_crypto_capability() failed\n"); + _ODP_ERR("odp_crypto_capability() failed\n"); return -1; } if (max_num_sa > crypto_capa.max_sessions) @@ -250,8 +250,7 @@ int _odp_ipsec_sad_term_global(void) if (odp_atomic_load_u32(&ipsec_sa->state) != IPSEC_SA_STATE_FREE) { - ODP_ERR("Not destroyed ipsec_sa: %u\n", - ipsec_sa->ipsec_sa_idx); + _ODP_ERR("Not destroyed ipsec_sa: %u\n", ipsec_sa->ipsec_sa_idx); rc = -1; } odp_atomic_store_u32(&ipsec_sa->state, IPSEC_SA_STATE_FREE); @@ -259,7 +258,7 @@ int _odp_ipsec_sad_term_global(void) ret = odp_shm_free(ipsec_sa_tbl->shm); if (ret < 0) { - ODP_ERR("shm free failed"); + _ODP_ERR("shm free failed"); rc = -1; } @@ -338,7 +337,7 @@ ipsec_sa_t *_odp_ipsec_sa_use(odp_ipsec_sa_t sa) { ipsec_sa_t *ipsec_sa; - ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); + _ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); ipsec_sa = ipsec_sa_entry_from_hdl(sa); @@ -354,7 +353,7 @@ void _odp_ipsec_sa_unuse(ipsec_sa_t *ipsec_sa) odp_ipsec_sa_t sa; odp_ipsec_warn_t warn = { .all = 0 }; - ODP_ASSERT(NULL != ipsec_sa); + _ODP_ASSERT(NULL != ipsec_sa); queue = ipsec_sa->queue; sa = ipsec_sa->ipsec_sa_hdl; @@ -433,7 +432,7 @@ static uint32_t esp_block_len_to_mask(uint32_t block_len) if (block_len < 4) block_len = 4; - ODP_ASSERT(_ODP_CHECK_IS_POWER2(block_len)); + _ODP_ASSERT(_ODP_CHECK_IS_POWER2(block_len)); return block_len - 1; } @@ -444,8 +443,8 @@ static int ipsec_antireplay_init(ipsec_sa_t *ipsec_sa, uint16_t num_bkts = 0; if (param->inbound.antireplay_ws > IPSEC_AR_WIN_SIZE_MAX) { - ODP_ERR("Anti-replay window size %" PRIu32 " is not supported.\n", - param->inbound.antireplay_ws); + _ODP_ERR("Anti-replay window size %" PRIu32 " is not supported.\n", + param->inbound.antireplay_ws); return -1; } @@ -499,7 +498,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) ipsec_sa = ipsec_sa_reserve(); if (NULL == ipsec_sa) { - ODP_ERR("No more free SA\n"); + _ODP_ERR("No more free SA\n"); return ODP_IPSEC_SA_INVALID; } @@ -734,13 +733,12 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) if (ipsec_sa->salt_length) { if (ipsec_sa->salt_length > IPSEC_MAX_SALT_LEN) { - ODP_ERR("IPSEC_MAX_SALT_LEN too small\n"); + _ODP_ERR("IPSEC_MAX_SALT_LEN too small\n"); goto error; } if (ipsec_sa->salt_length != salt_param->length) { - ODP_ERR("Bad extra keying material length: %i\n", - salt_param->length); + _ODP_ERR("Bad extra keying material length: %i\n", salt_param->length); goto error; } @@ -813,14 +811,13 @@ int odp_ipsec_sa_destroy(odp_ipsec_sa_t sa) uint32_t state = odp_atomic_load_u32(&ipsec_sa->state); if (IPSEC_SA_STATE_DISABLE != state) { - ODP_ERR("Distroying not disabled ipsec_sa: %u\n", - ipsec_sa->ipsec_sa_idx); + _ODP_ERR("Distroying not disabled ipsec_sa: %u\n", ipsec_sa->ipsec_sa_idx); return -1; } if (odp_crypto_session_destroy(ipsec_sa->session) < 0) { - ODP_ERR("Error destroying crypto session for ipsec_sa: %u\n", - ipsec_sa->ipsec_sa_idx); + _ODP_ERR("Error destroying crypto session for ipsec_sa: %u\n", + ipsec_sa->ipsec_sa_idx); rc = -1; } @@ -846,7 +843,7 @@ int odp_ipsec_sa_mtu_update(odp_ipsec_sa_t sa, uint32_t mtu) ipsec_sa_t *ipsec_sa; ipsec_sa = ipsec_sa_entry_from_hdl(sa); - ODP_ASSERT(NULL != ipsec_sa); + _ODP_ASSERT(NULL != ipsec_sa); odp_atomic_store_u32(&ipsec_sa->out.mtu, mtu); return 0; } @@ -1235,8 +1232,8 @@ int odp_ipsec_sa_info(odp_ipsec_sa_t sa, odp_ipsec_sa_info_t *sa_info) ipsec_sa = _odp_ipsec_sa_entry_from_hdl(sa); - ODP_ASSERT(ipsec_sa != NULL); - ODP_ASSERT(sa_info != NULL); + _ODP_ASSERT(ipsec_sa != NULL); + _ODP_ASSERT(sa_info != NULL); memset(sa_info, 0, sizeof(*sa_info)); param = &sa_info->param; diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 0f7c48f35..2c4f364e3 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -258,14 +258,14 @@ static int hp_create_file(uint64_t len, const char *filename) void *addr; if (len <= 0) { - ODP_ERR("Length is wrong\n"); + _ODP_ERR("Length is wrong\n"); return -1; } fd = open(filename, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if (fd < 0) { - ODP_ERR("Could not create cache file %s\n", filename); + _ODP_ERR("Could not create cache file %s\n", filename); return -1; } @@ -275,13 +275,13 @@ static int hp_create_file(uint64_t len, const char *filename) ret = fallocate(fd, 0, 0, len); if (ret == -1) { if (errno == ENOTSUP) { - ODP_DBG("fallocate() not supported\n"); + _ODP_DBG("fallocate() not supported\n"); ret = ftruncate(fd, len); } if (ret == -1) { - ODP_ERR("memory allocation failed: fd=%d, err=%s.\n", - fd, strerror(errno)); + _ODP_ERR("memory allocation failed: fd=%d, err=%s.\n", + fd, strerror(errno)); close(fd); return -1; } @@ -296,7 +296,7 @@ static int hp_create_file(uint64_t len, const char *filename) } _odp_ishmphy_unmap(addr, len, 0); - ODP_DBG("Created HP cache file %s, fd: %d\n", filename, fd); + _ODP_DBG("Created HP cache file %s, fd: %d\n", filename, fd); return fd; } @@ -316,10 +316,10 @@ static void hp_init(void) if (count <= 0) return; - ODP_DBG("Init HP cache with up to %d pages\n", count); + _ODP_DBG("Init HP cache with up to %d pages\n", count); if (!odp_global_ro.hugepage_info.default_huge_page_dir) { - ODP_ERR("No huge page dir\n"); + _ODP_ERR("No huge page dir\n"); return; } @@ -329,7 +329,7 @@ static void hp_init(void) if (mkdir(dir, 0744) != 0) { if (errno != EEXIST) { - ODP_ERR("Failed to create dir: %s\n", strerror(errno)); + _ODP_ERR("Failed to create dir: %s\n", strerror(errno)); return; } } @@ -343,7 +343,7 @@ static void hp_init(void) sizeof(struct huge_page_cache) + sizeof(int) * count, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); if (addr == MAP_FAILED) { - ODP_ERR("Unable to mmap memory for huge page cache\n."); + _ODP_ERR("Unable to mmap memory for huge page cache\n."); return; } @@ -369,8 +369,8 @@ static void hp_init(void) } hpc->idx = hpc->total - 1; - ODP_DBG("HP cache has %d huge pages of size 0x%08" PRIx64 "\n", - hpc->total, hpc->len); + _ODP_DBG("HP cache has %d huge pages of size 0x%08" PRIx64 "\n", + hpc->total, hpc->len); } static void hp_term(void) @@ -407,12 +407,12 @@ static int hp_get_cached(uint64_t len) static int hp_put_cached(int fd) { if (hpc == NULL) { - ODP_ERR("Bad hpc state\n"); + _ODP_ERR("Bad hpc state\n"); return -1; } if (odp_unlikely((hpc->idx + 1) >= hpc->total)) { - ODP_ERR("Trying to put more FD than allowed: %d\n", fd); + _ODP_ERR("Trying to put more FD than allowed: %d\n", fd); return -1; } @@ -463,7 +463,7 @@ static void *alloc_fragment(uintptr_t size, int block_index, intptr_t align, } if (!(*best_fragmnt)) { - ODP_ERR("Out of single VA memory. Try increasing " + _ODP_ERR("Out of single VA memory. Try increasing " "'shm.single_va_size_kb' in ODP config.\n"); return NULL; } @@ -480,7 +480,7 @@ static void *alloc_fragment(uintptr_t size, int block_index, intptr_t align, /* fragment space, i.e. take a new fragment descriptor... */ rem_fragmnt = ishm_ftbl->unused_fragmnts; if (!rem_fragmnt) { - ODP_ERR("unable to get shmem fragment descriptor!\n."); + _ODP_ERR("unable to get shmem fragment descriptor!\n."); return NULL; } ishm_ftbl->unused_fragmnts = rem_fragmnt->next; @@ -510,7 +510,7 @@ static void *alloc_fragment(uintptr_t size, int block_index, intptr_t align, /* otherwise, fragment space, i.e. take a new fragment descriptor... */ rem_fragmnt = ishm_ftbl->unused_fragmnts; if (!rem_fragmnt) { - ODP_ERR("unable to get shmem fragment descriptor!\n."); + _ODP_ERR("unable to get shmem fragment descriptor!\n."); return (*best_fragmnt)->start; } ishm_ftbl->unused_fragmnts = rem_fragmnt->next; @@ -562,7 +562,7 @@ static void free_fragment(ishm_fragment_t *fragmnt) if (ishm_ftbl->used_fragmnts == prev_f) ishm_ftbl->used_fragmnts = fragmnt; else - ODP_ERR("corrupted fragment list!.\n"); + _ODP_ERR("corrupted fragment list!.\n"); } fragmnt->prev = prev_f->prev; @@ -607,8 +607,7 @@ static int create_export_file(ishm_block_t *new_block, const char *name, name); export_file = fopen(new_block->exptname, "w"); if (export_file == NULL) { - ODP_ERR("open failed: err=%s.\n", - strerror(errno)); + _ODP_ERR("open failed: err=%s.\n", strerror(errno)); new_block->exptname[0] = 0; return -1; } @@ -688,7 +687,7 @@ static int create_file(int block_index, huge_flag_t huge, uint64_t len, fd = open(filename, oflag, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if (fd < 0) { if (huge != HUGE) - ODP_ERR("Normal page open failed: file=%s, " + _ODP_ERR("Normal page open failed: file=%s, " "err=\"%s\"\n", filename, strerror(errno)); return -1; } @@ -696,12 +695,12 @@ static int create_file(int block_index, huge_flag_t huge, uint64_t len, ret = fallocate(fd, 0, 0, len); if (ret == -1) { if (errno == ENOTSUP) { - ODP_DBG("fallocate() not supported\n"); + _ODP_DBG("fallocate() not supported\n"); ret = ftruncate(fd, len); } if (ret == -1) { - ODP_ERR("%s memory allocation failed: fd=%d, file=%s, " + _ODP_ERR("%s memory allocation failed: fd=%d, file=%s, " "err=\"%s\"\n", (huge == HUGE) ? "Huge page" : "Normal page", fd, filename, strerror(errno)); close(fd); @@ -758,7 +757,7 @@ static void *do_map(int block_index, uint64_t len, uint32_t align, ishm_block_t *new_block; /* entry in the main block table */ void *mapped_addr; - ODP_ASSERT(!(flags & _ODP_ISHM_SINGLE_VA)); + _ODP_ASSERT(!(flags & _ODP_ISHM_SINGLE_VA)); new_block = &ishm_tbl->block[block_index]; @@ -827,7 +826,7 @@ static void *alloc_single_va(const char *name, int new_index, uint64_t size, /* Allocate memory from the pre-reserved single VA space */ addr = alloc_fragment(len, new_index, align, &fragment); if (!addr) { - ODP_ERR("alloc_fragment failed.\n"); + _ODP_ERR("alloc_fragment failed.\n"); return NULL; } new_block->fragment = fragment; @@ -865,7 +864,7 @@ static void *do_remap(int block_index, int fd) offset = ishm_tbl->block[block_index].offset; flags = ishm_tbl->block[block_index].flags; - ODP_ASSERT(!(flags & _ODP_ISHM_SINGLE_VA)); + _ODP_ASSERT(!(flags & _ODP_ISHM_SINGLE_VA)); /* try to mmap: */ mapped_addr = _odp_ishmphy_map(fd, len, offset, flags); @@ -988,7 +987,7 @@ static int block_free_internal(int block_index, int close_fd, int deregister) if ((block_index < 0) || (block_index >= ISHM_MAX_NB_BLOCKS) || (ishm_tbl->block[block_index].len == 0)) { - ODP_ERR("Request to free an invalid block\n"); + _ODP_ERR("Request to free an invalid block\n"); return -1; } @@ -1084,7 +1083,7 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, /* check if we have reached the maximum number of allocation: */ if (new_index >= ISHM_MAX_NB_BLOCKS) { odp_spinlock_unlock(&ishm_tbl->lock); - ODP_ERR("ISHM_MAX_NB_BLOCKS limit reached!\n"); + _ODP_ERR("ISHM_MAX_NB_BLOCKS limit reached!\n"); return -1; } @@ -1112,7 +1111,7 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, &fd); if (addr == NULL) { odp_spinlock_unlock(&ishm_tbl->lock); - ODP_ERR("_ishm_reserve failed.\n"); + _ODP_ERR("_ishm_reserve failed.\n"); return -1; } new_block->huge = EXTERNAL; @@ -1149,8 +1148,7 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, addr = do_map(new_index, len, hp_align, 0, flags, CACHED, &fd); if (addr == NULL) { - ODP_ERR("Could not use cached hp %d\n", - fd); + _ODP_ERR("Could not use cached hp %d\n", fd); hp_put_cached(fd); fd = -1; } else { @@ -1163,9 +1161,9 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, if (addr == NULL) { if (!huge_error_printed) { - ODP_ERR("No huge pages, fall back to " - "normal pages. Check: " - "/proc/sys/vm/nr_hugepages.\n"); + _ODP_ERR("No huge pages, fall back to " + "normal pages. Check: " + "/proc/sys/vm/nr_hugepages.\n"); huge_error_printed = 1; } } else { @@ -1178,7 +1176,7 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, if (fd < 0) { if (user_flags & ODP_SHM_HP) { odp_spinlock_unlock(&ishm_tbl->lock); - ODP_ERR("Unable to allocate memory from huge pages\n"); + _ODP_ERR("Unable to allocate memory from huge pages\n"); return -1; } /* at least, alignment in VA should match page size, but user @@ -1216,7 +1214,7 @@ use_single_va: } delete_file(new_block); odp_spinlock_unlock(&ishm_tbl->lock); - ODP_ERR("_ishm_reserve failed.\n"); + _ODP_ERR("_ishm_reserve failed.\n"); return -1; } @@ -1277,7 +1275,7 @@ static void *reserve_single_va(uint64_t size, int *fd_out) } } if (fd < 0) - ODP_ERR("No huge pages, fall back to normal pages. " + _ODP_ERR("No huge pages, fall back to normal pages. " "Check: /proc/sys/vm/nr_hugepages.\n"); ishm_tbl->single_va_huge = true; } @@ -1299,7 +1297,7 @@ static void *reserve_single_va(uint64_t size, int *fd_out) close(fd); unlink(ishm_tbl->single_va_filename); } - ODP_ERR("Reserving single VA memory failed.\n"); + _ODP_ERR("Reserving single VA memory failed.\n"); return NULL; } @@ -1339,7 +1337,7 @@ int _odp_ishm_find_exported(const char *remote_name, pid_t external_odp_pid, export_file = fopen(export_filename, "r"); if (export_file == NULL) { - ODP_ERR("Error opening %s.\n", export_filename); + _ODP_ERR("Error opening %s.\n", export_filename); return -1; } @@ -1375,8 +1373,7 @@ int _odp_ishm_find_exported(const char *remote_name, pid_t external_odp_pid, /* now open the filename given in the description file: */ fd = open(filename, O_RDWR, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if (fd == -1) { - ODP_ERR("open failed for %s: %s.\n", - filename, strerror(errno)); + _ODP_ERR("open failed for %s: %s.\n", filename, strerror(errno)); return -1; } @@ -1404,7 +1401,7 @@ int _odp_ishm_find_exported(const char *remote_name, pid_t external_odp_pid, error_exp_file: fclose(export_file); - ODP_ERR("Error reading %s.\n", export_filename); + _ODP_ERR("Error reading %s.\n", export_filename); return -1; } @@ -1454,7 +1451,7 @@ static void *block_lookup(int block_index) if ((block_index < 0) || (block_index >= ISHM_MAX_NB_BLOCKS) || (ishm_tbl->block[block_index].len == 0)) { - ODP_ERR("Request to lookup an invalid block\n"); + _ODP_ERR("Request to lookup an invalid block\n"); return NULL; } @@ -1466,7 +1463,7 @@ static void *block_lookup(int block_index) /* this ishm is not known by this process, yet: we create the mapping.*/ fd = _odp_fdserver_lookup_fd(FD_SRV_CTX_ISHM, block_index); if (fd < 0) { - ODP_ERR("Could not find ishm file descriptor (BUG!)\n"); + _ODP_ERR("Could not find ishm file descriptor (BUG!)\n"); return NULL; } @@ -1480,7 +1477,7 @@ static void *block_lookup(int block_index) mapped_addr = do_remap(block_index, fd); if (mapped_addr == NULL) { - ODP_ERR(" lookup: Could not map existing shared memory!\n"); + _ODP_ERR(" lookup: Could not map existing shared memory!\n"); return NULL; } @@ -1545,7 +1542,7 @@ int _odp_ishm_info(int block_index, _odp_ishm_info_t *info) (block_index >= ISHM_MAX_NB_BLOCKS) || (ishm_tbl->block[block_index].len == 0)) { odp_spinlock_unlock(&ishm_tbl->lock); - ODP_ERR("Request for info on an invalid block\n"); + _ODP_ERR("Request for info on an invalid block\n"); return -1; } @@ -1631,8 +1628,7 @@ int _odp_ishm_cleanup_files(const char *dirpath) dir = opendir(userdir); if (!dir) { /* ok if the dir does not exist. no much to delete then! */ - ODP_DBG("opendir failed for %s: %s\n", - userdir, strerror(errno)); + _ODP_DBG("opendir failed for %s: %s\n", userdir, strerror(errno)); return 0; } snprintf(prefix, PATH_MAX, _ODP_FILES_FMT, odp_global_ro.main_pid); @@ -1647,10 +1643,9 @@ int _odp_ishm_cleanup_files(const char *dirpath) } snprintf(fullpath, PATH_MAX, "%s/%s", userdir, e->d_name); - ODP_DBG("deleting obsolete file: %s\n", fullpath); + _ODP_DBG("deleting obsolete file: %s\n", fullpath); if (unlink(fullpath)) - ODP_ERR("unlink failed for %s: %s\n", - fullpath, strerror(errno)); + _ODP_ERR("unlink failed for %s: %s\n", fullpath, strerror(errno)); free(fullpath); } } @@ -1673,23 +1668,23 @@ int _odp_ishm_init_global(const odp_init_t *init) if (!_odp_libconfig_lookup_ext_int("shm", NULL, "single_va_size_kb", &val_kb)) { - ODP_ERR("Unable to read single VA size from config\n"); + _ODP_ERR("Unable to read single VA size from config\n"); return -1; } - ODP_DBG("Shm single VA size: %dkB\n", val_kb); + _ODP_DBG("Shm single VA size: %dkB\n", val_kb); max_memory = (uint64_t)val_kb * 1024; internal = max_memory / 8; if (!_odp_libconfig_lookup_ext_int("shm", NULL, "huge_page_limit_kb", &val_kb)) { - ODP_ERR("Unable to read huge page usage limit from config\n"); + _ODP_ERR("Unable to read huge page usage limit from config\n"); return -1; } huge_page_limit = (uint64_t)val_kb * 1024; - ODP_DBG("Shm huge page usage limit: %dkB\n", val_kb); + _ODP_DBG("Shm huge page usage limit: %dkB\n", val_kb); /* user requested memory size + some extra for internal use */ if (init && init->shm.max_memory) @@ -1709,22 +1704,22 @@ int _odp_ishm_init_global(const odp_init_t *init) odp_global_ro.shm_dir_from_env = 0; } - ODP_DBG("ishm: using dir %s\n", odp_global_ro.shm_dir); + _ODP_DBG("ishm: using dir %s\n", odp_global_ro.shm_dir); uid = getuid(); snprintf(odp_global_ro.uid, UID_MAXLEN, "%d", uid); if ((syscall(SYS_gettid)) != odp_global_ro.main_pid) { - ODP_ERR("ishm init must be performed by the main " + _ODP_ERR("ishm init must be performed by the main " "ODP process!\n."); return -1; } if (!hp_dir) { - ODP_DBG("NOTE: No support for huge pages\n"); + _ODP_DBG("NOTE: No support for huge pages\n"); } else { - ODP_DBG("Huge pages mount point is: %s\n", hp_dir); + _ODP_DBG("Huge pages mount point is: %s\n", hp_dir); _odp_ishm_cleanup_files(hp_dir); } @@ -1734,7 +1729,7 @@ int _odp_ishm_init_global(const odp_init_t *init) addr = mmap(NULL, sizeof(ishm_table_t), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); if (addr == MAP_FAILED) { - ODP_ERR("unable to mmap the main block table\n."); + _ODP_ERR("unable to mmap the main block table\n."); goto init_glob_err1; } ishm_tbl = addr; @@ -1748,7 +1743,7 @@ int _odp_ishm_init_global(const odp_init_t *init) addr = mmap(NULL, sizeof(ishm_ftable_t), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); if (addr == MAP_FAILED) { - ODP_ERR("unable to mmap the main fragment table\n."); + _ODP_ERR("unable to mmap the main fragment table\n."); goto init_glob_err2; } ishm_ftbl = addr; @@ -1760,7 +1755,7 @@ int _odp_ishm_init_global(const odp_init_t *init) spce_addr = reserve_single_va(max_memory, &ishm_tbl->single_va_fd); if (!spce_addr) { - ODP_ERR("unable to reserve single VA memory\n."); + _ODP_ERR("unable to reserve single VA memory\n."); goto init_glob_err3; } ishm_tbl->single_va_start = spce_addr; @@ -1790,7 +1785,7 @@ int _odp_ishm_init_global(const odp_init_t *init) * indeed assume the availability of odp_shm_reserve()...: */ if (do_odp_ishm_init_local()) { - ODP_ERR("unable to init the main thread\n."); + _ODP_ERR("unable to init the main thread\n."); goto init_glob_err4; } @@ -1804,13 +1799,13 @@ int _odp_ishm_init_global(const odp_init_t *init) init_glob_err4: if (_odp_ishmphy_free_single_va()) - ODP_ERR("unable to free single VA memory\n."); + _ODP_ERR("unable to free single VA memory\n."); init_glob_err3: if (munmap(ishm_ftbl, sizeof(ishm_ftable_t)) < 0) - ODP_ERR("unable to munmap main fragment table\n."); + _ODP_ERR("unable to munmap main fragment table\n."); init_glob_err2: if (munmap(ishm_tbl, sizeof(ishm_table_t)) < 0) - ODP_ERR("unable to munmap main block table\n."); + _ODP_ERR("unable to munmap main block table\n."); init_glob_err1: return -1; } @@ -1866,7 +1861,7 @@ static int do_odp_ishm_term_local(void) block = &ishm_tbl->block[block_index]; if ((--block->refcnt) <= 0) { block->refcnt = 0; - ODP_DBG("Warning: block %d: name:%s " + _ODP_DBG("Warning: block %d: name:%s " "no longer referenced\n", i, ishm_tbl->block[i].name[0] ? @@ -1907,14 +1902,14 @@ int _odp_ishm_term_global(void) if ((getpid() != odp_global_ro.main_pid) || (syscall(SYS_gettid) != getpid())) - ODP_ERR("odp_term_global() must be performed by the main " + _ODP_ERR("odp_term_global() must be performed by the main " "ODP process!\n."); /* cleanup possibly non freed memory (and complain a bit): */ for (index = 0; index < ISHM_MAX_NB_BLOCKS; index++) { block = &ishm_tbl->block[index]; if (block->len != 0) { - ODP_ERR("block '%s' (file %s) was never freed " + _ODP_ERR("block '%s' (file %s) was never freed " "(cleaning up...).\n", block->name, block->filename); delete_file(block); @@ -1930,12 +1925,12 @@ int _odp_ishm_term_global(void) /* free the fragment table */ if (munmap(ishm_ftbl, sizeof(ishm_ftable_t)) < 0) { ret |= -1; - ODP_ERR("unable to munmap fragment table\n."); + _ODP_ERR("unable to munmap fragment table\n."); } /* free the block table */ if (munmap(ishm_tbl, sizeof(ishm_table_t)) < 0) { ret |= -1; - ODP_ERR("unable to munmap main table\n."); + _ODP_ERR("unable to munmap main table\n."); } /* free the reserved single VA memory */ @@ -1943,7 +1938,7 @@ int _odp_ishm_term_global(void) ret |= -1; if ((fd >= 0) && close(fd)) { ret |= -1; - ODP_ERR("unable to close single VA\n."); + _ODP_ERR("unable to close single VA\n."); } if (!odp_global_ro.shm_dir_from_env) @@ -1995,10 +1990,10 @@ int _odp_ishm_status(const char *title) max_name_len = str_len; } - ODP_PRINT("%s\n", title); - ODP_PRINT(" %-*s flag %-29s %-8s %-8s %-3s %-3s %-3s file\n", - max_name_len, "name", "range", "user_len", "unused", - "seq", "ref", "fd"); + _ODP_PRINT("%s\n", title); + _ODP_PRINT(" %-*s flag %-29s %-8s %-8s %-3s %-3s %-3s file\n", + max_name_len, "name", "range", "user_len", "unused", + "seq", "ref", "fd"); /* display block table: 1 line per entry +1 extra line if mapped here */ for (i = 0; i < ISHM_MAX_NB_BLOCKS; i++) { @@ -2046,7 +2041,7 @@ int _odp_ishm_status(const char *title) entry_fd = ishm_proctable->entry[proc_index].fd; } - ODP_PRINT("%2i %-*s %s%c %p-%p %-8" PRIu64 " " + _ODP_PRINT("%2i %-*s %s%c %p-%p %-8" PRIu64 " " "%-8" PRIu64 " %-3" PRIu64 " %-3" PRIu64 " " "%-3d %s\n", i, max_name_len, ishm_tbl->block[i].name, @@ -2060,39 +2055,39 @@ int _odp_ishm_status(const char *title) ishm_tbl->block[i].filename : "(none)"); } - ODP_PRINT("TOTAL: %58s%-8" PRIu64 " %2s%-8" PRIu64 "\n", - "", len_total, - "", lost_total); - ODP_PRINT("%65s(%" PRIu64 "MB) %4s(%" PRIu64 "MB)\n", - "", len_total / 1024 / 1024, - "", lost_total / 1024 / 1024); + _ODP_PRINT("TOTAL: %58s%-8" PRIu64 " %2s%-8" PRIu64 "\n", + "", len_total, + "", lost_total); + _ODP_PRINT("%65s(%" PRIu64 "MB) %4s(%" PRIu64 "MB)\n", + "", len_total / 1024 / 1024, + "", lost_total / 1024 / 1024); /* display the virtual space allocations... : */ - ODP_PRINT("\nishm virtual space:\n"); + _ODP_PRINT("\nishm virtual space:\n"); for (fragmnt = ishm_ftbl->used_fragmnts; fragmnt; fragmnt = fragmnt->next) { if (fragmnt->block_index >= 0) { nb_allocated_frgments++; - ODP_PRINT(" %8p - %8p: ALLOCATED by block:%d\n", - fragmnt->start, - (void *)((uintptr_t)fragmnt->start + fragmnt->len - 1), - fragmnt->block_index); + _ODP_PRINT(" %8p - %8p: ALLOCATED by block:%d\n", + fragmnt->start, + (void *)((uintptr_t)fragmnt->start + fragmnt->len - 1), + fragmnt->block_index); consecutive_unallocated = 0; } else { - ODP_PRINT(" %8p - %8p: NOT ALLOCATED\n", - fragmnt->start, - (void *)((uintptr_t)fragmnt->start + fragmnt->len - 1)); + _ODP_PRINT(" %8p - %8p: NOT ALLOCATED\n", + fragmnt->start, + (void *)((uintptr_t)fragmnt->start + fragmnt->len - 1)); if (consecutive_unallocated++) - ODP_ERR("defragmentation error\n"); + _ODP_ERR("defragmentation error\n"); } /* some other sanity checks: */ if (fragmnt->prev != previous) - ODP_ERR("chaining error\n"); + _ODP_ERR("chaining error\n"); if (fragmnt != ishm_ftbl->used_fragmnts) { if ((uintptr_t)fragmnt->start != last_address + 1) - ODP_ERR("lost space error\n"); + _ODP_ERR("lost space error\n"); } last_address = (uintptr_t)fragmnt->start + fragmnt->len - 1; @@ -2105,24 +2100,24 @@ int _odp_ishm_status(const char *title) * the number of used fragments: */ if (single_va_blocks != nb_allocated_frgments) - ODP_ERR("single_va_blocks != nb_allocated_fragments!\n"); + _ODP_ERR("single_va_blocks != nb_allocated_fragments!\n"); /* compute the number of unused fragments*/ for (fragmnt = ishm_ftbl->unused_fragmnts; fragmnt; fragmnt = fragmnt->next) nb_unused_frgments++; - ODP_PRINT("ishm: %d fragment used. %d fragments unused. (total=%d)\n", - nb_used_frgments, nb_unused_frgments, - nb_used_frgments + nb_unused_frgments); + _ODP_PRINT("ishm: %d fragment used. %d fragments unused. (total=%d)\n", + nb_used_frgments, nb_unused_frgments, + nb_used_frgments + nb_unused_frgments); if ((nb_used_frgments + nb_unused_frgments) != ISHM_NB_FRAGMNTS) - ODP_ERR("lost fragments!\n"); + _ODP_ERR("lost fragments!\n"); if (nb_blocks < ishm_proctable->nb_entries) - ODP_ERR("process known block cannot exceed main total sum!\n"); + _ODP_ERR("process known block cannot exceed main total sum!\n"); - ODP_PRINT("\n"); + _ODP_PRINT("\n"); odp_spinlock_unlock(&ishm_tbl->lock); return nb_blocks; @@ -2139,21 +2134,21 @@ void _odp_ishm_print(int block_index) (block_index >= ISHM_MAX_NB_BLOCKS) || (ishm_tbl->block[block_index].len == 0)) { odp_spinlock_unlock(&ishm_tbl->lock); - ODP_ERR("Request for info on an invalid block\n"); + _ODP_ERR("Request for info on an invalid block\n"); return; } block = &ishm_tbl->block[block_index]; - ODP_PRINT("\nSHM block info\n--------------\n"); - ODP_PRINT(" name: %s\n", block->name); - ODP_PRINT(" file: %s\n", block->filename); - ODP_PRINT(" expt: %s\n", block->exptname); - ODP_PRINT(" user_flags: 0x%x\n", block->user_flags); - ODP_PRINT(" flags: 0x%x\n", block->flags); - ODP_PRINT(" user_len: %" PRIu64 "\n", block->user_len); - ODP_PRINT(" start: %p\n", block->start); - ODP_PRINT(" len: %" PRIu64 "\n", block->len); + _ODP_PRINT("\nSHM block info\n--------------\n"); + _ODP_PRINT(" name: %s\n", block->name); + _ODP_PRINT(" file: %s\n", block->filename); + _ODP_PRINT(" expt: %s\n", block->exptname); + _ODP_PRINT(" user_flags: 0x%x\n", block->user_flags); + _ODP_PRINT(" flags: 0x%x\n", block->flags); + _ODP_PRINT(" user_len: %" PRIu64 "\n", block->user_len); + _ODP_PRINT(" start: %p\n", block->start); + _ODP_PRINT(" len: %" PRIu64 "\n", block->len); switch (block->huge) { case HUGE: @@ -2172,10 +2167,10 @@ void _odp_ishm_print(int block_index) str = "??"; } - ODP_PRINT(" page type: %s\n", str); - ODP_PRINT(" seq: %" PRIu64 "\n", block->seq); - ODP_PRINT(" refcnt: %" PRIu64 "\n", block->refcnt); - ODP_PRINT("\n"); + _ODP_PRINT(" page type: %s\n", str); + _ODP_PRINT(" seq: %" PRIu64 "\n", block->seq); + _ODP_PRINT(" refcnt: %" PRIu64 "\n", block->refcnt); + _ODP_PRINT("\n"); odp_spinlock_unlock(&ishm_tbl->lock); } diff --git a/platform/linux-generic/odp_ishmphy.c b/platform/linux-generic/odp_ishmphy.c index efaf12c7b..796482b98 100644 --- a/platform/linux-generic/odp_ishmphy.c +++ b/platform/linux-generic/odp_ishmphy.c @@ -51,14 +51,14 @@ void *_odp_ishmphy_reserve_single_va(uint64_t len, int fd) addr = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, 0); if (addr == MAP_FAILED) { - ODP_ERR("mmap failed: %s\n", strerror(errno)); + _ODP_ERR("mmap failed: %s\n", strerror(errno)); return NULL; } if (mprotect(addr, len, PROT_READ | PROT_WRITE)) - ODP_ERR("mprotect failed: %s\n", strerror(errno)); + _ODP_ERR("mprotect failed: %s\n", strerror(errno)); - ODP_DBG("VA Reserved: %p, len=%" PRIu64 "\n", addr, len); + _ODP_DBG("VA Reserved: %p, len=%" PRIu64 "\n", addr, len); common_va_address = addr; common_va_len = len; @@ -79,7 +79,7 @@ int _odp_ishmphy_free_single_va(void) ret = munmap(common_va_address, common_va_len); if (ret) - ODP_ERR("munmap failed: %s\n", strerror(errno)); + _ODP_ERR("munmap failed: %s\n", strerror(errno)); return ret; } @@ -94,7 +94,7 @@ void *_odp_ishmphy_map(int fd, uint64_t size, uint64_t offset, int flags) void *mapped_addr; int mmap_flags = MAP_POPULATE; - ODP_ASSERT(!(flags & _ODP_ISHM_SINGLE_VA)); + _ODP_ASSERT(!(flags & _ODP_ISHM_SINGLE_VA)); /* do a new mapping in the VA space: */ mapped_addr = mmap(NULL, size, PROT_READ | PROT_WRITE, @@ -102,7 +102,7 @@ void *_odp_ishmphy_map(int fd, uint64_t size, uint64_t offset, int flags) if ((mapped_addr >= common_va_address) && ((char *)mapped_addr < (char *)common_va_address + common_va_len)) { - ODP_ERR("VA SPACE OVERLAP!\n"); + _ODP_ERR("VA SPACE OVERLAP!\n"); } if (mapped_addr == MAP_FAILED) @@ -111,9 +111,9 @@ void *_odp_ishmphy_map(int fd, uint64_t size, uint64_t offset, int flags) /* if locking is requested, lock it...*/ if (flags & _ODP_ISHM_LOCK) { if (mlock(mapped_addr, size)) { - ODP_ERR("mlock failed: %s\n", strerror(errno)); + _ODP_ERR("mlock failed: %s\n", strerror(errno)); if (munmap(mapped_addr, size)) - ODP_ERR("munmap failed: %s\n", strerror(errno)); + _ODP_ERR("munmap failed: %s\n", strerror(errno)); return NULL; } } @@ -140,6 +140,6 @@ int _odp_ishmphy_unmap(void *start, uint64_t len, int flags) /* just release the mapping */ ret = munmap(start, len); if (ret) - ODP_ERR("munmap failed: %s\n", strerror(errno)); + _ODP_ERR("munmap failed: %s\n", strerror(errno)); return ret; } diff --git a/platform/linux-generic/odp_ishmpool.c b/platform/linux-generic/odp_ishmpool.c index 4186444fa..5bcb2c2ea 100644 --- a/platform/linux-generic/odp_ishmpool.c +++ b/platform/linux-generic/odp_ishmpool.c @@ -157,7 +157,7 @@ static inline void remove_from_list(pool_t *bpool, uint8_t order, } remove_from_list_error: - ODP_ERR("List corrupted\n"); + _ODP_ERR("List corrupted\n"); } /* @@ -227,13 +227,13 @@ static pool_t *_odp_ishmbud_pool_create(const char *pool_name, int store_idx, blk_idx = _odp_ishm_reserve(pool_name, total_sz, -1, ODP_CACHE_LINE_SIZE, 0, flags, 0); if (blk_idx < 0) { - ODP_ERR("_odp_ishm_reserve failed."); + _ODP_ERR("_odp_ishm_reserve failed."); return NULL; } bpool = _odp_ishm_address(blk_idx); if (bpool == NULL) { - ODP_ERR("_odp_ishm_address failed."); + _ODP_ERR("_odp_ishm_address failed."); return NULL; } @@ -284,7 +284,7 @@ static void *_odp_ishmbud_alloc(pool_t *bpool, uint64_t size) /* if size is zero or too big reject: */ if ((!size) && (size > (1ULL << bpool->ctrl.order))) { - ODP_ERR("Invalid alloc size (0 or larger than whole pool)\n"); + _ODP_ERR("Invalid alloc size (0 or larger than whole pool)\n"); return NULL; } @@ -314,7 +314,7 @@ static void *_odp_ishmbud_alloc(pool_t *bpool, uint64_t size) if (!bblock) { odp_spinlock_unlock(&bpool->ctrl.lock); - ODP_ERR("Out of memory. (Buddy pool full)\n"); + _ODP_ERR("Out of memory. (Buddy pool full)\n"); return NULL; } @@ -366,7 +366,7 @@ static int _odp_ishmbud_free(pool_t *bpool, void *addr) if (((uintptr_t)addr < user_start) || ((uintptr_t)addr > user_stop) || (((uintptr_t)addr - user_start) & mask)) { - ODP_ERR("Invalid address to be freed\n"); + _ODP_ERR("Invalid address to be freed\n"); return -1; } @@ -378,7 +378,7 @@ static int _odp_ishmbud_free(pool_t *bpool, void *addr) nr = get_bblock_nr(bpool, bblock); order = bpool->ctrl.alloced_order[nr]; if (order == BBLOCK_FREE) { - ODP_ERR("Double free error\n"); + _ODP_ERR("Double free error\n"); odp_spinlock_unlock(&bpool->ctrl.lock); return -1; } @@ -470,13 +470,13 @@ static pool_t *_odp_ishmslab_pool_create(const char *pool_name, int store_idx, blk_idx = _odp_ishm_reserve(pool_name, total_sz, -1, ODP_CACHE_LINE_SIZE, 0, flags, 0); if (blk_idx < 0) { - ODP_ERR("_odp_ishm_reserve failed."); + _ODP_ERR("_odp_ishm_reserve failed."); return NULL; } spool = _odp_ishm_address(blk_idx); if (spool == NULL) { - ODP_ERR("_odp_ishm_address failed."); + _ODP_ERR("_odp_ishm_address failed."); return NULL; } @@ -525,7 +525,7 @@ static void *_odp_ishmslab_alloc(pool_t *spool, uint64_t size) ret = spool->ctrl.free_head; if (!ret) { odp_spinlock_unlock(&spool->ctrl.lock); - ODP_ERR("Out of memory. (Slab pool full)\n"); + _ODP_ERR("Out of memory. (Slab pool full)\n"); return NULL; } @@ -555,7 +555,7 @@ static int _odp_ishmslab_free(pool_t *spool, void *addr) if (((uintptr_t)addr < user_start) || ((uintptr_t)addr > user_stop) || (((uintptr_t)addr - user_start) % spool->ctrl.element_sz)) { - ODP_ERR("Invalid address to be freed\n"); + _ODP_ERR("Invalid address to be freed\n"); return -1; } @@ -579,7 +579,7 @@ pool_t *_odp_ishm_pool_create(const char *pool_name, uint64_t size, uint64_t real_pool_sz; if (min_alloc > max_alloc) { - ODP_ERR("invalid parameter: min_alloc > max_alloc"); + _ODP_ERR("invalid parameter: min_alloc > max_alloc"); return NULL; } @@ -589,7 +589,7 @@ pool_t *_odp_ishm_pool_create(const char *pool_name, uint64_t size, break; } if (store_idx == MAX_NB_POOL) { - ODP_ERR("Max number of pool reached (MAX_NB_POOL)"); + _ODP_ERR("Max number of pool reached (MAX_NB_POOL)"); return NULL; } diff --git a/platform/linux-generic/odp_libconfig.c b/platform/linux-generic/odp_libconfig.c index 87a41deb8..d5e159e21 100644 --- a/platform/linux-generic/odp_libconfig.c +++ b/platform/linux-generic/odp_libconfig.c @@ -34,9 +34,9 @@ int _odp_libconfig_init_global(void) odp_global_ro.has_config_rt = 0; if (!config_read_string(config, config_builtin)) { - ODP_ERR("Failed to read default config: %s(%d): %s\n", - config_error_file(config), config_error_line(config), - config_error_text(config)); + _ODP_ERR("Failed to read default config: %s(%d): %s\n", + config_error_file(config), config_error_line(config), + config_error_text(config)); goto fail; } @@ -44,46 +44,44 @@ int _odp_libconfig_init_global(void) if (filename == NULL) return 0; - ODP_PRINT("ODP CONFIG FILE: %s\n", filename); + _ODP_PRINT("ODP CONFIG FILE: %s\n", filename); if (!config_read_file(config_rt, filename)) { - ODP_PRINT(" ERROR: failed to read config file: %s(%d): %s\n\n", - config_error_file(config_rt), - config_error_line(config_rt), - config_error_text(config_rt)); + _ODP_PRINT(" ERROR: failed to read config file: %s(%d): %s\n\n", + config_error_file(config_rt), + config_error_line(config_rt), + config_error_text(config_rt)); goto fail; } /* Check runtime configuration's implementation name and version */ if (!config_lookup_string(config, impl_field, &impl) || !config_lookup_string(config_rt, impl_field, &impl_rt)) { - ODP_PRINT(" ERROR: missing mandatory field: %s\n\n", - impl_field); + _ODP_PRINT(" ERROR: missing mandatory field: %s\n\n", impl_field); goto fail; } if (!config_lookup_string(config, vers_field, &vers) || !config_lookup_string(config_rt, vers_field, &vers_rt)) { - ODP_PRINT(" ERROR: missing mandatory field: %s\n\n", - vers_field); + _ODP_PRINT(" ERROR: missing mandatory field: %s\n\n", vers_field); goto fail; } if (strcmp(impl, impl_rt)) { - ODP_PRINT(" ERROR: ODP implementation name mismatch:\n" - " Expected: \"%s\"\n" - " Found: \"%s\"\n\n", impl, impl_rt); + _ODP_PRINT(" ERROR: ODP implementation name mismatch:\n" + " Expected: \"%s\"\n" + " Found: \"%s\"\n\n", impl, impl_rt); goto fail; } if (strcmp(vers, vers_rt)) { - ODP_PRINT(" ERROR: config file version number mismatch:\n" - " Expected: \"%s\"\n" - " Found: \"%s\"\n\n", vers, vers_rt); + _ODP_PRINT(" ERROR: config file version number mismatch:\n" + " Expected: \"%s\"\n" + " Found: \"%s\"\n\n", vers, vers_rt); goto fail; } odp_global_ro.has_config_rt = 1; return 0; fail: - ODP_ERR("Config file failure\n"); + _ODP_ERR("Config file failure\n"); config_destroy(config); config_destroy(config_rt); return -1; @@ -133,8 +131,7 @@ int _odp_libconfig_lookup_str(const char *path, char *value, return length; if (length > str_size) { - ODP_ERR("libconfig: length of %d bigger than size %u\n", - length, str_size); + _ODP_ERR("libconfig: length of %d bigger than size %u\n", length, str_size); return -1; } @@ -142,7 +139,7 @@ int _odp_libconfig_lookup_str(const char *path, char *value, return length; } - ODP_ERR("libconfig: %s is not defined in config files\n", path); + _ODP_ERR("libconfig: %s is not defined in config files\n", path); return -1; } @@ -199,7 +196,7 @@ int _odp_libconfig_lookup_array_str(const char *path, char **value, /* invalid config if element is not an array */ if (config_setting_is_array(setting) == CONFIG_FALSE) { - ODP_ERR("libconfig: %s is not an array\n", path); + _ODP_ERR("libconfig: %s is not an array\n", path); return -1; } num = config_setting_length(setting); @@ -209,15 +206,15 @@ int _odp_libconfig_lookup_array_str(const char *path, char **value, elem = config_setting_get_elem(setting, 0); if (config_setting_type(elem) != CONFIG_TYPE_STRING) { - ODP_ERR("libconfig: %s array is not of type string\n", path); + _ODP_ERR("libconfig: %s array is not of type string\n", path); return -1; } for (j = 0; j < num; j++) { elem = config_setting_get_elem(setting, j); if (strlen(elem->value.sval) > str_size) { - ODP_ERR("libconfig: length of %s bigger than size %u\n", - elem->value.sval, str_size); + _ODP_ERR("libconfig: length of %s bigger than size %u\n", + elem->value.sval, str_size); return -1; } strcpy(value[j], elem->value.sval); @@ -226,7 +223,7 @@ int _odp_libconfig_lookup_array_str(const char *path, char **value, return num; } - ODP_ERR("libconfig: %s is not defined in config files\n", path); + _ODP_ERR("libconfig: %s is not defined in config files\n", path); return -1; } @@ -336,7 +333,7 @@ int _odp_libconfig_print(void) /* Print temp file to the log */ rewind(file); while ((c = fgetc(file)) != EOF) - ODP_PRINT("%c", (char)c); + _ODP_PRINT("%c", (char)c); fclose(file); return 0; diff --git a/platform/linux-generic/odp_name_table.c b/platform/linux-generic/odp_name_table.c index fbb35cf00..7c93d594b 100644 --- a/platform/linux-generic/odp_name_table.c +++ b/platform/linux-generic/odp_name_table.c @@ -223,20 +223,19 @@ static void secondary_hash_dump(secondary_hash_tbl_t *secondary_hash_tbl) entry_cnt = hash_tbl_entry & 0x3F; list_cnt = linked_list_len(name_tbl_entry); if (entry_cnt != list_cnt) - ODP_DBG("%s idx=%u entry_cnt=%u " + _ODP_DBG("%s idx=%u entry_cnt=%u " "list_cnt=%u\n", __func__, idx, entry_cnt, list_cnt); count += entry_cnt; } else { - ODP_DBG("%s inner secondary tbl\n", - __func__); + _ODP_DBG("%s inner secondary tbl\n", __func__); } } } - ODP_DBG("%s count=%u\n", __func__, count); + _ODP_DBG("%s count=%u\n", __func__, count); } static uint32_t name_tbl_free_list_add(name_tbl_t *name_tbl, @@ -1073,7 +1072,7 @@ static void secondary_hash_histo_print(void) if (name_hash_tbl.num_secondary_tbls[0] == 0) return; - ODP_DBG(" level1 secondary hash histogram:\n"); + _ODP_DBG(" level1 secondary hash histogram:\n"); total_count = 0; for (idx = 0; idx < 256; idx++) { count = level1_histo[idx]; @@ -1081,24 +1080,23 @@ static void secondary_hash_histo_print(void) total_count += count * idx; if (count != 0) - ODP_DBG(" num collisions=%02u count=%u\n", - idx, count); + _ODP_DBG(" num collisions=%02u count=%u\n", idx, count); } count = level1_histo[256]; total_count += count; if (count != 0) - ODP_DBG(" num collisions >=256 count=%u\n", count); + _ODP_DBG(" num collisions >=256 count=%u\n", count); avg = (100 * total_count) / name_hash_tbl.num_secondary_tbls[0]; avg = avg / SECONDARY_HASH_TBL_SIZE; - ODP_DBG(" avg collisions=%02u.%02u total=%u\n\n", - avg / 100, avg % 100, total_count); + _ODP_DBG(" avg collisions=%02u.%02u total=%u\n\n", + avg / 100, avg % 100, total_count); if (name_hash_tbl.num_secondary_tbls[1] == 0) return; - ODP_DBG(" level2 secondary hash histogram:\n"); + _ODP_DBG(" level2 secondary hash histogram:\n"); total_count = 0; for (idx = 0; idx < 256; idx++) { count = level2_histo[idx]; @@ -1106,19 +1104,18 @@ static void secondary_hash_histo_print(void) total_count += count * idx; if (count != 0) - ODP_DBG(" num collisions=%02u count=%u\n", - idx, count); + _ODP_DBG(" num collisions=%02u count=%u\n", idx, count); } count = level2_histo[256]; total_count += count; if (count != 0) - ODP_DBG(" num collisions >=256 count=%u\n", count); + _ODP_DBG(" num collisions >=256 count=%u\n", count); avg = (100 * total_count) / name_hash_tbl.num_secondary_tbls[1]; avg = avg / SECONDARY_HASH_TBL_SIZE; - ODP_DBG(" avg collisions=%02u.%02u total=%u\n\n", - avg / 100, avg % 100, total_count); + _ODP_DBG(" avg collisions=%02u.%02u total=%u\n\n", + avg / 100, avg % 100, total_count); } #endif @@ -1130,21 +1127,21 @@ void _odp_int_name_tbl_stats_print(void) count, total_count; uint32_t avg; - ODP_DBG("\nname table stats:\n"); - ODP_DBG(" num_names=%" PRIu32 " num_adds=%" PRIu64 " " - "num_deletes=%" PRIu64 " num_name_tbls=%" PRIu8 "\n", - name_tbls.current_num_names, name_tbls.num_adds, - name_tbls.num_deletes, name_tbls.num_name_tbls); + _ODP_DBG("\nname table stats:\n"); + _ODP_DBG(" num_names=%" PRIu32 " num_adds=%" PRIu64 " " + "num_deletes=%" PRIu64 " num_name_tbls=%" PRIu8 "\n", + name_tbls.current_num_names, name_tbls.num_adds, + name_tbls.num_deletes, name_tbls.num_name_tbls); for (idx = 0; idx < NUM_NAME_TBLS; idx++) { name_tbl = name_tbls.tbls[idx]; if ((name_tbl) && (name_tbl->num_used != 0)) - ODP_DBG(" name_tbl %u num_allocd=%7u " - "num_added_to_free_list=%7u " - "num_used=%7u num_avail_to_add=%7u\n", idx, - name_tbl->num_allocd, - name_tbl->num_added_to_free_list, - name_tbl->num_used, - name_tbl->num_avail_to_add); + _ODP_DBG(" name_tbl %u num_allocd=%7u " + "num_added_to_free_list=%7u " + "num_used=%7u num_avail_to_add=%7u\n", idx, + name_tbl->num_allocd, + name_tbl->num_added_to_free_list, + name_tbl->num_used, + name_tbl->num_avail_to_add); } memset(primary_hash_histo, 0, sizeof(primary_hash_histo)); @@ -1154,7 +1151,7 @@ void _odp_int_name_tbl_stats_print(void) primary_hash_histo[collisions]++; } - ODP_DBG(" name_tbl primary hash histogram:\n"); + _ODP_DBG(" name_tbl primary hash histogram:\n"); total_count = 0; for (idx = 0; idx < 256; idx++) { count = primary_hash_histo[idx]; @@ -1162,23 +1159,20 @@ void _odp_int_name_tbl_stats_print(void) total_count += count * idx; if (count != 0) - ODP_DBG(" num collisions=%02u count=%u\n", - idx, count); + _ODP_DBG(" num collisions=%02u count=%u\n", idx, count); } count = primary_hash_histo[256]; total_count += count; if (count != 0) - ODP_DBG(" num collisions >=256 count=%u\n", count); + _ODP_DBG(" num collisions >=256 count=%u\n", count); avg = (100 * total_count) / PRIMARY_HASH_TBL_SIZE; - ODP_DBG(" avg collisions=%02u.%02u total=%u\n\n", - avg / 100, avg % 100, total_count); + _ODP_DBG(" avg collisions=%02u.%02u total=%u\n\n", + avg / 100, avg % 100, total_count); - ODP_DBG(" num of first level secondary hash tbls=%u " - "second level tbls=%u\n", - name_hash_tbl.num_secondary_tbls[0], - name_hash_tbl.num_secondary_tbls[1]); + _ODP_DBG(" num of first level secondary hash tbls=%u second level tbls=%u\n", + name_hash_tbl.num_secondary_tbls[0], name_hash_tbl.num_secondary_tbls[1]); #ifdef SECONDARY_HASH_HISTO_PRINT if (name_hash_tbl.num_secondary_tbls[0] != 0) diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 775836b66..2ebf602e8 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -25,6 +25,7 @@ #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> #include <odp_pool_internal.h> +#include <odp_print_internal.h> /* Inlined API functions */ #include <odp/api/plat/byteorder_inlines.h> @@ -65,7 +66,8 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .timestamp = offsetof(odp_packet_hdr_t, timestamp), .input_flags = offsetof(odp_packet_hdr_t, p.input_flags), .flags = offsetof(odp_packet_hdr_t, p.flags), - .subtype = offsetof(odp_packet_hdr_t, subtype) + .subtype = offsetof(odp_packet_hdr_t, subtype), + .cls_mark = offsetof(odp_packet_hdr_t, cls_mark) }; @@ -290,7 +292,7 @@ static inline void link_segments(odp_packet_hdr_t *pkt_hdr[], int num) ref_cnt = &pkt_hdr[cur]->ref_cnt; prev_ref = odp_atomic_fetch_inc_u32(ref_cnt); - ODP_ASSERT(prev_ref == 0); + _ODP_ASSERT(prev_ref == 0); } cur++; @@ -325,7 +327,7 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) uint32_t prev_ref = odp_atomic_fetch_inc_u32(&hdr->ref_cnt); - ODP_ASSERT(prev_ref == 0); + _ODP_ASSERT(prev_ref == 0); } /* Link segments */ @@ -693,7 +695,7 @@ void odp_packet_free(odp_packet_t pkt) odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); int num_seg = pkt_hdr->seg_count; - ODP_ASSERT(segment_ref(pkt_hdr) > 0); + _ODP_ASSERT(segment_ref(pkt_hdr) > 0); if (odp_likely(num_seg == 1)) packet_free_multi(&pkt_hdr, 1); @@ -711,7 +713,7 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num) odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]); int num_seg = pkt_hdr->seg_count; - ODP_ASSERT(segment_ref(pkt_hdr) > 0); + _ODP_ASSERT(segment_ref(pkt_hdr) > 0); if (odp_unlikely(num_seg > 1)) { free_all_segments(pkt_hdr, num_seg); @@ -904,7 +906,7 @@ void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) if (len > pkt_hdr->tailroom) return NULL; - ODP_ASSERT(odp_packet_has_ref(pkt) == 0); + _ODP_ASSERT(odp_packet_has_ref(pkt) == 0); old_tail = packet_tail(pkt_hdr); push_tail(pkt_hdr, len); @@ -921,7 +923,7 @@ int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, uint32_t tail_off = frame_len; int ret = 0; - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); + _ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); if (len > tailroom) { pool_t *pool = _odp_pool_entry(pkt_hdr->event_hdr.pool); @@ -958,7 +960,7 @@ void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); odp_packet_hdr_t *last_seg = packet_last_seg(pkt_hdr); - ODP_ASSERT(odp_packet_has_ref(pkt) == 0); + _ODP_ASSERT(odp_packet_has_ref(pkt) == 0); if (len >= last_seg->seg_len) return NULL; @@ -979,7 +981,7 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, if (len >= pkt_hdr->frame_len) return -1; - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); + _ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); last = pkt_hdr->seg_count - 1; last_seg = packet_last_seg(pkt_hdr); @@ -1026,53 +1028,6 @@ void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len, * */ -void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ptr) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (odp_unlikely(ptr == NULL)) { - pkt_hdr->p.flags.user_ptr_set = 0; - return; - } - - pkt_hdr->user_ptr = ptr; - pkt_hdr->p.flags.user_ptr_set = 1; -} - -int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (offset >= pkt_hdr->frame_len) - return -1; - - packet_hdr_has_l2_set(pkt_hdr, 1); - pkt_hdr->p.l2_offset = offset; - return 0; -} - -int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (offset >= pkt_hdr->frame_len) - return -1; - - pkt_hdr->p.l3_offset = offset; - return 0; -} - -int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (offset >= pkt_hdr->frame_len) - return -1; - - pkt_hdr->p.l4_offset = offset; - return 0; -} - uint16_t odp_packet_ones_comp(odp_packet_t pkt, odp_packet_data_range_t *range) { (void)pkt; @@ -1081,48 +1036,6 @@ uint16_t odp_packet_ones_comp(odp_packet_t pkt, odp_packet_data_range_t *range) return 0; } -void odp_packet_l3_chksum_insert(odp_packet_t pkt, int insert) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - pkt_hdr->p.flags.l3_chksum_set = 1; - pkt_hdr->p.flags.l3_chksum = insert; -} - -void odp_packet_l4_chksum_insert(odp_packet_t pkt, int insert) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - pkt_hdr->p.flags.l4_chksum_set = 1; - pkt_hdr->p.flags.l4_chksum = insert; -} - -odp_packet_chksum_status_t odp_packet_l3_chksum_status(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (!pkt_hdr->p.input_flags.l3_chksum_done) - return ODP_PACKET_CHKSUM_UNKNOWN; - - if (pkt_hdr->p.flags.l3_chksum_err) - return ODP_PACKET_CHKSUM_BAD; - - return ODP_PACKET_CHKSUM_OK; -} - -odp_packet_chksum_status_t odp_packet_l4_chksum_status(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (!pkt_hdr->p.input_flags.l4_chksum_done) - return ODP_PACKET_CHKSUM_UNKNOWN; - - if (pkt_hdr->p.flags.l4_chksum_err) - return ODP_PACKET_CHKSUM_BAD; - - return ODP_PACKET_CHKSUM_OK; -} - void odp_packet_flow_hash_set(odp_packet_t pkt, uint32_t flow_hash) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); @@ -1130,13 +1043,6 @@ void odp_packet_flow_hash_set(odp_packet_t pkt, uint32_t flow_hash) packet_set_flow_hash(pkt_hdr, flow_hash); } -void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - packet_set_ts(pkt_hdr, ×tamp); -} - /* * * Segment level @@ -1231,7 +1137,7 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t offset, uint32_t len, if (align > ODP_CACHE_LINE_SIZE) return -1; - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); + _ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); if (seglen >= len) { misalign = align <= 1 ? 0 : @@ -1270,10 +1176,10 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) uint32_t dst_len = dst_hdr->frame_len; uint32_t src_len = src_hdr->frame_len; - ODP_ASSERT(odp_packet_has_ref(*dst) == 0); + _ODP_ASSERT(odp_packet_has_ref(*dst) == 0); if (odp_unlikely(dst_len + src_len > dst_pool->max_len)) { - ODP_ERR("concat would result oversized packet\n"); + _ODP_ERR("concat would result oversized packet\n"); return -1; } @@ -1311,7 +1217,7 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) if (len >= pktlen || tail == NULL) return -1; - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); + _ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); *tail = odp_packet_copy_part(*pkt, len, pktlen - len, odp_packet_pool(*pkt)); @@ -1338,7 +1244,7 @@ odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_pool_t pool) md_copy = _odp_packet_copy_md_possible(pool, odp_packet_pool(pkt)); if (odp_unlikely(md_copy < 0)) { - ODP_ERR("Unable to copy packet metadata\n"); + _ODP_ERR("Unable to copy packet metadata\n"); return ODP_PACKET_INVALID; } @@ -1472,7 +1378,7 @@ int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset, int ret; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - ODP_ASSERT(offset + len <= pkt_hdr->frame_len); + _ODP_ASSERT(offset + len <= pkt_hdr->frame_len); while (len > 0) { mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); @@ -1499,31 +1405,31 @@ static int packet_print_input_flags(odp_packet_hdr_t *hdr, char *str, int max) int len = 0; if (hdr->p.input_flags.l2) - len += snprintf(&str[len], max - len, "l2 "); + len += _odp_snprint(&str[len], max - len, "l2 "); if (hdr->p.input_flags.l3) - len += snprintf(&str[len], max - len, "l3 "); + len += _odp_snprint(&str[len], max - len, "l3 "); if (hdr->p.input_flags.l4) - len += snprintf(&str[len], max - len, "l4 "); + len += _odp_snprint(&str[len], max - len, "l4 "); if (hdr->p.input_flags.eth) - len += snprintf(&str[len], max - len, "eth "); + len += _odp_snprint(&str[len], max - len, "eth "); if (hdr->p.input_flags.vlan) - len += snprintf(&str[len], max - len, "vlan "); + len += _odp_snprint(&str[len], max - len, "vlan "); if (hdr->p.input_flags.arp) - len += snprintf(&str[len], max - len, "arp "); + len += _odp_snprint(&str[len], max - len, "arp "); if (hdr->p.input_flags.ipv4) - len += snprintf(&str[len], max - len, "ipv4 "); + len += _odp_snprint(&str[len], max - len, "ipv4 "); if (hdr->p.input_flags.ipv6) - len += snprintf(&str[len], max - len, "ipv6 "); + len += _odp_snprint(&str[len], max - len, "ipv6 "); if (hdr->p.input_flags.ipsec) - len += snprintf(&str[len], max - len, "ipsec "); + len += _odp_snprint(&str[len], max - len, "ipsec "); if (hdr->p.input_flags.udp) - len += snprintf(&str[len], max - len, "udp "); + len += _odp_snprint(&str[len], max - len, "udp "); if (hdr->p.input_flags.tcp) - len += snprintf(&str[len], max - len, "tcp "); + len += _odp_snprint(&str[len], max - len, "tcp "); if (hdr->p.input_flags.sctp) - len += snprintf(&str[len], max - len, "sctp "); + len += _odp_snprint(&str[len], max - len, "sctp "); if (hdr->p.input_flags.icmp) - len += snprintf(&str[len], max - len, "icmp "); + len += _odp_snprint(&str[len], max - len, "icmp "); return len; } @@ -1537,40 +1443,45 @@ void odp_packet_print(odp_packet_t pkt) int n = max_len - 1; odp_packet_hdr_t *hdr = packet_hdr(pkt); - len += snprintf(&str[len], n - len, "Packet\n------\n"); - len += snprintf(&str[len], n - len, " pool index %u\n", hdr->event_hdr.index.pool); - len += snprintf(&str[len], n - len, " buf index %u\n", hdr->event_hdr.index.event); - len += snprintf(&str[len], n - len, " ev subtype %i\n", hdr->subtype); - len += snprintf(&str[len], n - len, " input_flags 0x%" PRIx64 "\n", - hdr->p.input_flags.all); + len += _odp_snprint(&str[len], n - len, "Packet info\n"); + len += _odp_snprint(&str[len], n - len, "-----------\n"); + len += _odp_snprint(&str[len], n - len, " handle 0x%" PRIx64 "\n", + odp_packet_to_u64(pkt)); + len += _odp_snprint(&str[len], n - len, " pool index %u\n", hdr->event_hdr.index.pool); + len += _odp_snprint(&str[len], n - len, " buf index %u\n", + hdr->event_hdr.index.event); + len += _odp_snprint(&str[len], n - len, " ev subtype %i\n", hdr->subtype); + len += _odp_snprint(&str[len], n - len, " input_flags 0x%" PRIx64 "\n", + hdr->p.input_flags.all); if (hdr->p.input_flags.all) { - len += snprintf(&str[len], n - len, " "); + len += _odp_snprint(&str[len], n - len, " "); len += packet_print_input_flags(hdr, &str[len], n - len); - len += snprintf(&str[len], n - len, "\n"); - } - len += snprintf(&str[len], n - len, " flags 0x%" PRIx32 "\n", - hdr->p.flags.all_flags); - len += snprintf(&str[len], n - len, " cls_mark %" PRIu64 "\n", - odp_packet_cls_mark(pkt)); - len += snprintf(&str[len], n - len, - " l2_offset %" PRIu32 "\n", hdr->p.l2_offset); - len += snprintf(&str[len], n - len, - " l3_offset %" PRIu32 "\n", hdr->p.l3_offset); - len += snprintf(&str[len], n - len, - " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); - len += snprintf(&str[len], n - len, - " frame_len %" PRIu32 "\n", hdr->frame_len); - len += snprintf(&str[len], n - len, - " input %" PRIu64 "\n", - odp_pktio_to_u64(hdr->input)); - len += snprintf(&str[len], n - len, - " headroom %" PRIu32 "\n", - odp_packet_headroom(pkt)); - len += snprintf(&str[len], n - len, - " tailroom %" PRIu32 "\n", - odp_packet_tailroom(pkt)); - len += snprintf(&str[len], n - len, - " num_segs %i\n", odp_packet_num_segs(pkt)); + len += _odp_snprint(&str[len], n - len, "\n"); + } + len += _odp_snprint(&str[len], n - len, + " flags 0x%" PRIx32 "\n", hdr->p.flags.all_flags); + len += _odp_snprint(&str[len], n - len, + " cls_mark %" PRIu64 "\n", odp_packet_cls_mark(pkt)); + len += _odp_snprint(&str[len], n - len, + " user ptr %p\n", hdr->user_ptr); + len += _odp_snprint(&str[len], n - len, + " user area %p\n", hdr->uarea_addr); + len += _odp_snprint(&str[len], n - len, + " l2_offset %" PRIu32 "\n", hdr->p.l2_offset); + len += _odp_snprint(&str[len], n - len, + " l3_offset %" PRIu32 "\n", hdr->p.l3_offset); + len += _odp_snprint(&str[len], n - len, + " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); + len += _odp_snprint(&str[len], n - len, + " frame_len %" PRIu32 "\n", hdr->frame_len); + len += _odp_snprint(&str[len], n - len, + " input %" PRIu64 "\n", odp_pktio_to_u64(hdr->input)); + len += _odp_snprint(&str[len], n - len, + " headroom %" PRIu32 "\n", odp_packet_headroom(pkt)); + len += _odp_snprint(&str[len], n - len, + " tailroom %" PRIu32 "\n", odp_packet_tailroom(pkt)); + len += _odp_snprint(&str[len], n - len, + " num_segs %i\n", odp_packet_num_segs(pkt)); seg = odp_packet_first_seg(pkt); @@ -1579,25 +1490,22 @@ void odp_packet_print(odp_packet_t pkt) char seg_str[max_len]; int str_len; - str_len = snprintf(&seg_str[0], max_len, - " [%d] seg_len %-4" PRIu32 " seg_data %p " - " ref_cnt %u\n", - seg_idx, - odp_packet_seg_data_len(pkt, seg), - odp_packet_seg_data(pkt, seg), - segment_ref(seg_hdr)); + str_len = _odp_snprint(&seg_str[0], max_len, + " [%d] seg_len %-4" PRIu32 " seg_data %p ref_cnt %u\n", + seg_idx, odp_packet_seg_data_len(pkt, seg), + odp_packet_seg_data(pkt, seg), segment_ref(seg_hdr)); /* Prevent print buffer overflow */ if (n - len - str_len < 10) { - len += snprintf(&str[len], n - len, " ...\n"); + len += _odp_snprint(&str[len], n - len, " ...\n"); break; } - len += snprintf(&str[len], n - len, "%s", seg_str); + len += _odp_snprint(&str[len], n - len, "%s", seg_str); seg = odp_packet_next_seg(pkt, seg); } - ODP_PRINT("%s\n", str); + _ODP_PRINT("%s\n", str); } void odp_packet_print_data(odp_packet_t pkt, uint32_t offset, @@ -1613,26 +1521,28 @@ void odp_packet_print_data(odp_packet_t pkt, uint32_t offset, uint32_t data_len = odp_packet_len(pkt); pool_t *pool = _odp_pool_entry(hdr->event_hdr.pool); - len += snprintf(&str[len], n - len, "Packet\n------\n"); - len += snprintf(&str[len], n - len, - " pool index %" PRIu32 "\n", pool->pool_idx); - len += snprintf(&str[len], n - len, - " buf index %" PRIu32 "\n", - hdr->event_hdr.index.event); - len += snprintf(&str[len], n - len, - " seg_count %" PRIu16 "\n", hdr->seg_count); - len += snprintf(&str[len], n - len, - " data len %" PRIu32 "\n", data_len); - len += snprintf(&str[len], n - len, - " data ptr %p\n", odp_packet_data(pkt)); - len += snprintf(&str[len], n - len, - " print offset %" PRIu32 "\n", offset); - len += snprintf(&str[len], n - len, - " print length %" PRIu32 "\n", byte_len); + len += _odp_snprint(&str[len], n - len, "Packet data\n"); + len += _odp_snprint(&str[len], n - len, "-----------\n"); + len += _odp_snprint(&str[len], n - len, + " handle 0x%" PRIx64 "\n", odp_packet_to_u64(pkt)); + len += _odp_snprint(&str[len], n - len, + " pool index %" PRIu32 "\n", pool->pool_idx); + len += _odp_snprint(&str[len], n - len, + " buf index %" PRIu32 "\n", hdr->event_hdr.index.event); + len += _odp_snprint(&str[len], n - len, + " seg_count %" PRIu16 "\n", hdr->seg_count); + len += _odp_snprint(&str[len], n - len, + " data len %" PRIu32 "\n", data_len); + len += _odp_snprint(&str[len], n - len, + " data ptr %p\n", odp_packet_data(pkt)); + len += _odp_snprint(&str[len], n - len, + " print offset %" PRIu32 "\n", offset); + len += _odp_snprint(&str[len], n - len, + " print length %" PRIu32 "\n", byte_len); if (offset + byte_len > data_len) { - len += snprintf(&str[len], n - len, " BAD OFFSET OR LEN\n"); - ODP_PRINT("%s\n", str); + len += _odp_snprint(&str[len], n - len, " BAD OFFSET OR LEN\n"); + _ODP_PRINT("%s\n", str); return; } @@ -1648,18 +1558,18 @@ void odp_packet_print_data(odp_packet_t pkt, uint32_t offset, odp_packet_copy_to_mem(pkt, offset, copy_len, data); - len += snprintf(&str[len], n - len, " "); + len += _odp_snprint(&str[len], n - len, " "); for (i = 0; i < copy_len; i++) - len += snprintf(&str[len], n - len, " %02x", data[i]); + len += _odp_snprint(&str[len], n - len, " %02x", data[i]); - len += snprintf(&str[len], n - len, "\n"); + len += _odp_snprint(&str[len], n - len, "\n"); byte_len -= copy_len; offset += copy_len; } - ODP_PRINT("%s\n", str); + _ODP_PRINT("%s\n", str); } int odp_packet_is_valid(odp_packet_t pkt) @@ -1957,7 +1867,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, if (sum != 0) { pkt_hdr->p.flags.l4_chksum_err = 1; pkt_hdr->p.flags.udp_err = 1; - ODP_DBG("UDP chksum fail (%x)!\n", sum); + _ODP_DBG("UDP chksum fail (%x)!\n", sum); if (opt.bit.drop_udp_err) return -1; } @@ -1977,7 +1887,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, if (sum != 0) { pkt_hdr->p.flags.l4_chksum_err = 1; pkt_hdr->p.flags.tcp_err = 1; - ODP_DBG("TCP chksum fail (%x)!\n", sum); + _ODP_DBG("TCP chksum fail (%x)!\n", sum); if (opt.bit.drop_tcp_err) return -1; } @@ -2009,8 +1919,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, if (sum != sctp->chksum) { pkt_hdr->p.flags.l4_chksum_err = 1; pkt_hdr->p.flags.sctp_err = 1; - ODP_DBG("SCTP chksum fail (%x/%x)!\n", sum, - sctp->chksum); + _ODP_DBG("SCTP chksum fail (%x/%x)!\n", sum, sctp->chksum); if (opt.bit.drop_sctp_err) return -1; } @@ -2025,6 +1934,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); const uint8_t *data; uint32_t seg_len; + uint32_t seg_end; uint32_t packet_len = pkt_hdr->frame_len; odp_proto_t proto = param->proto; odp_proto_layer_t layer = param->last_layer; @@ -2057,6 +1967,8 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, data = buf; } + seg_end = offset + seg_len; /* one past the maximum offset */ + /* Reset parser flags, keep other flags */ packet_parse_reset(pkt_hdr, 0); @@ -2080,7 +1992,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, opt.bit.sctp_chksum = param->chksums.chksum.sctp; ret = _odp_packet_parse_common_l3_l4(&pkt_hdr->p, data, offset, - packet_len, seg_len, layer, + packet_len, seg_end, layer, ethtype, &l4_part_sum, opt); if (ret) @@ -2190,14 +2102,14 @@ odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) new = odp_packet_copy(pkt, odp_packet_pool(pkt)); if (new == ODP_PACKET_INVALID) { - ODP_ERR("copy failed\n"); + _ODP_ERR("copy failed\n"); return ODP_PACKET_INVALID; } ret = odp_packet_trunc_head(&new, offset, NULL, NULL); if (ret < 0) { - ODP_ERR("trunk_head failed\n"); + _ODP_ERR("trunk_head failed\n"); odp_packet_free(new); return ODP_PACKET_INVALID; } @@ -2214,14 +2126,14 @@ odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, ref = odp_packet_ref(pkt, offset); if (ref == ODP_PACKET_INVALID) { - ODP_DBG("reference create failed\n"); + _ODP_DBG("reference create failed\n"); return ODP_PACKET_INVALID; } ret = odp_packet_concat(&hdr, ref); if (ret < 0) { - ODP_DBG("concat failed\n"); + _ODP_DBG("concat failed\n"); odp_packet_free(ref); return ODP_PACKET_INVALID; } @@ -2246,73 +2158,6 @@ int odp_packet_has_ref(odp_packet_t pkt) return 0; } -odp_proto_l2_type_t odp_packet_l2_type(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (pkt_hdr->p.input_flags.eth) - return ODP_PROTO_L2_TYPE_ETH; - - return ODP_PROTO_L2_TYPE_NONE; -} - -odp_proto_l3_type_t odp_packet_l3_type(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (pkt_hdr->p.input_flags.ipv4) - return ODP_PROTO_L3_TYPE_IPV4; - else if (pkt_hdr->p.input_flags.ipv6) - return ODP_PROTO_L3_TYPE_IPV6; - else if (pkt_hdr->p.input_flags.arp) - return ODP_PROTO_L3_TYPE_ARP; - - return ODP_PROTO_L3_TYPE_NONE; -} - -odp_proto_l4_type_t odp_packet_l4_type(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (pkt_hdr->p.input_flags.tcp) - return ODP_PROTO_L4_TYPE_TCP; - else if (pkt_hdr->p.input_flags.udp) - return ODP_PROTO_L4_TYPE_UDP; - else if (pkt_hdr->p.input_flags.sctp) - return ODP_PROTO_L4_TYPE_SCTP; - else if (pkt_hdr->p.input_flags.ipsec_ah) - return ODP_PROTO_L4_TYPE_AH; - else if (pkt_hdr->p.input_flags.ipsec_esp) - return ODP_PROTO_L4_TYPE_ESP; - else if (pkt_hdr->p.input_flags.icmp && - pkt_hdr->p.input_flags.ipv4) - return ODP_PROTO_L4_TYPE_ICMPV4; - else if (pkt_hdr->p.input_flags.icmp && - pkt_hdr->p.input_flags.ipv6) - return ODP_PROTO_L4_TYPE_ICMPV6; - else if (pkt_hdr->p.input_flags.no_next_hdr) - return ODP_PROTO_L4_TYPE_NO_NEXT; - - return ODP_PROTO_L4_TYPE_NONE; -} - -uint64_t odp_packet_cls_mark(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (pkt_hdr->p.input_flags.cls_mark) - return pkt_hdr->cls_mark; - - return 0; -} - -void odp_packet_ts_request(odp_packet_t pkt, int enable) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - pkt_hdr->p.flags.ts_set = !!enable; -} - void odp_packet_lso_request_clr(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); @@ -2382,7 +2227,7 @@ int odp_packet_has_tx_compl_request(odp_packet_t pkt) void odp_packet_tx_compl_free(odp_packet_tx_compl_t tx_compl) { if (odp_unlikely(tx_compl == ODP_PACKET_TX_COMPL_INVALID)) { - ODP_ERR("Bad TX completion event handle\n"); + _ODP_ERR("Bad TX completion event handle\n"); return; } @@ -2392,7 +2237,7 @@ void odp_packet_tx_compl_free(odp_packet_tx_compl_t tx_compl) void *odp_packet_tx_compl_user_ptr(odp_packet_tx_compl_t tx_compl) { if (odp_unlikely(tx_compl == ODP_PACKET_TX_COMPL_INVALID)) { - ODP_ERR("Bad TX completion event handle\n"); + _ODP_ERR("Bad TX completion event handle\n"); return NULL; } @@ -2434,17 +2279,17 @@ uint32_t odp_packet_disassemble(odp_packet_t pkt, odp_packet_buf_t pkt_buf[], ui uint32_t num_segs = odp_packet_num_segs(pkt); if (odp_unlikely(pool->type != ODP_POOL_PACKET)) { - ODP_ERR("Not a packet pool\n"); + _ODP_ERR("Not a packet pool\n"); return 0; } if (odp_unlikely(pool->pool_ext == 0)) { - ODP_ERR("Not an external memory pool\n"); + _ODP_ERR("Not an external memory pool\n"); return 0; } if (odp_unlikely(num < num_segs)) { - ODP_ERR("Not enough buffer handles %u. Packet has %u segments.\n", num, num_segs); + _ODP_ERR("Not enough buffer handles %u. Packet has %u segments.\n", num, num_segs); return 0; } @@ -2468,17 +2313,17 @@ odp_packet_t odp_packet_reassemble(odp_pool_t pool_hdl, odp_packet_buf_t pkt_buf pool_t *pool = _odp_pool_entry(pool_hdl); if (odp_unlikely(pool->type != ODP_POOL_PACKET)) { - ODP_ERR("Not a packet pool\n"); + _ODP_ERR("Not a packet pool\n"); return ODP_PACKET_INVALID; } if (odp_unlikely(pool->pool_ext == 0)) { - ODP_ERR("Not an external memory pool\n"); + _ODP_ERR("Not an external memory pool\n"); return ODP_PACKET_INVALID; } if (odp_unlikely(num == 0)) { - ODP_ERR("Bad number of buffers: %u\n", num); + _ODP_ERR("Bad number of buffers: %u\n", num); return ODP_PACKET_INVALID; } diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index c9f43ab9c..794452127 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -84,37 +84,37 @@ static int read_config_file(pktio_global_t *pktio_glb) const char *str; int val = 0; - ODP_PRINT("Packet IO config:\n"); + _ODP_PRINT("Packet IO config:\n"); str = "pktio.pktin_frame_offset"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } if (val < 0 || val > UINT16_MAX) { - ODP_ERR("Bad value %s = %i\n", str, val); + _ODP_ERR("Bad value %s = %i\n", str, val); return -1; } pktio_glb->config.pktin_frame_offset = val; - ODP_PRINT(" %s: %i\n", str, val); + _ODP_PRINT(" %s: %i\n", str, val); str = "pktio.tx_compl_pool_size"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } if (val < 0) { - ODP_ERR("Bad value %s = %i\n", str, val); + _ODP_ERR("Bad value %s = %i\n", str, val); return -1; } pktio_glb->config.tx_compl_pool_size = val; - ODP_PRINT(" %s: %i\n", str, val); + _ODP_PRINT(" %s: %i\n", str, val); - ODP_PRINT("\n"); + _ODP_PRINT("\n"); return 0; } @@ -158,15 +158,14 @@ int _odp_pktio_init_global(void) for (pktio_if = 0; _odp_pktio_if_ops[pktio_if]; ++pktio_if) { if (_odp_pktio_if_ops[pktio_if]->init_global) if (_odp_pktio_if_ops[pktio_if]->init_global()) { - ODP_ERR("failed to initialized pktio type %d", - pktio_if); + _ODP_ERR("failed to initialized pktio type %d", pktio_if); return -1; } } if (_ODP_PCAPNG) { if (_odp_pcapng_init_global()) { - ODP_ERR("Failed to initialize pcapng\n"); + _ODP_ERR("Failed to initialize pcapng\n"); return -1; } } @@ -181,8 +180,7 @@ int _odp_pktio_init_local(void) for (pktio_if = 0; _odp_pktio_if_ops[pktio_if]; ++pktio_if) { if (_odp_pktio_if_ops[pktio_if]->init_local) if (_odp_pktio_if_ops[pktio_if]->init_local()) { - ODP_ERR("failed to initialized pktio type %d", - pktio_if); + _ODP_ERR("failed to initialized pktio type %d", pktio_if); return -1; } } @@ -304,7 +302,7 @@ static odp_pktio_t setup_pktio_entry(const char *name, odp_pool_t pool, if (strlen(name) >= PKTIO_NAME_LEN - 1) { /* ioctl names limitation */ - ODP_ERR("pktio name %s is too long (max: %d chars)\n", name, PKTIO_NAME_LEN - 1); + _ODP_ERR("pktio name %s is too long (max: %d chars)\n", name, PKTIO_NAME_LEN - 1); return ODP_PKTIO_INVALID; } @@ -322,7 +320,7 @@ static odp_pktio_t setup_pktio_entry(const char *name, odp_pool_t pool, } if (i == ODP_CONFIG_PKTIO_ENTRIES) { - ODP_ERR("All pktios used already\n"); + _ODP_ERR("All pktios used already\n"); return ODP_PKTIO_INVALID; } @@ -355,7 +353,7 @@ static odp_pktio_t setup_pktio_entry(const char *name, odp_pool_t pool, if (ret != 0) { pktio_entry->state = PKTIO_STATE_FREE; unlock_entry(pktio_entry); - ODP_ERR("Unable to init any I/O type.\n"); + _ODP_ERR("Unable to init any I/O type.\n"); return ODP_PKTIO_INVALID; } @@ -385,7 +383,7 @@ static const char *driver_name(odp_pktio_t hdl) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); + _ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return "bad handle"; } @@ -403,7 +401,7 @@ odp_pktio_t odp_pktio_open(const char *name, odp_pool_t pool, param = &default_param; } - ODP_ASSERT(pool_type_is_packet(pool)); + _ODP_ASSERT(pool_type_is_packet(pool)); hdl = odp_pktio_lookup(name); if (hdl != ODP_PKTIO_INVALID) { @@ -416,7 +414,7 @@ odp_pktio_t odp_pktio_open(const char *name, odp_pool_t pool, hdl = setup_pktio_entry(name, pool, param); odp_spinlock_unlock(&pktio_global->lock); - ODP_DBG("interface: %s, driver: %s\n", name, driver_name(hdl)); + _ODP_DBG("interface: %s, driver: %s\n", name, driver_name(hdl)); return hdl; } @@ -462,7 +460,7 @@ static void destroy_out_queues(pktio_entry_t *entry, int num) for (i = 0; i < num; i++) { if (entry->out_queue[i].queue != ODP_QUEUE_INVALID) { rc = odp_queue_destroy(entry->out_queue[i].queue); - ODP_ASSERT(rc == 0); + _ODP_ASSERT(rc == 0); entry->out_queue[i].queue = ODP_QUEUE_INVALID; } } @@ -486,7 +484,7 @@ static void flush_in_queues(pktio_entry_t *entry) while ((ret = odp_pktin_recv(pktin, packets, max_pkts))) { if (ret < 0) { - ODP_ERR("Queue flush failed\n"); + _ODP_ERR("Queue flush failed\n"); return; } @@ -503,12 +501,12 @@ int odp_pktio_close(odp_pktio_t hdl) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_ERR("Bad handle\n"); + _ODP_ERR("Bad handle\n"); return -1; } if (entry->state == PKTIO_STATE_STARTED) { - ODP_DBG("Missing odp_pktio_stop() before close.\n"); + _ODP_DBG("Missing odp_pktio_stop() before close.\n"); return -1; } @@ -526,7 +524,7 @@ int odp_pktio_close(odp_pktio_t hdl) if (entry->tx_compl_pool != ODP_POOL_INVALID) { if (odp_pool_destroy(entry->tx_compl_pool)) { unlock_entry(entry); - ODP_ERR("Unable to destroy Tx event completion pool\n"); + _ODP_ERR("Unable to destroy Tx event completion pool\n"); return -1; } } @@ -535,11 +533,11 @@ int odp_pktio_close(odp_pktio_t hdl) res = _pktio_close(entry); odp_spinlock_unlock(&pktio_global->lock); if (res) - ODP_ABORT("unable to close pktio\n"); + _ODP_ABORT("unable to close pktio\n"); unlock_entry(entry); - ODP_DBG("interface: %s\n", entry->name); + _ODP_DBG("interface: %s\n", entry->name); return 0; } @@ -577,7 +575,7 @@ int odp_pktio_config(odp_pktio_t hdl, const odp_pktio_config_t *config) entry = get_pktio_entry(hdl); if (!entry) { - ODP_ERR("Bad handle\n"); + _ODP_ERR("Bad handle\n"); return -1; } @@ -591,23 +589,23 @@ int odp_pktio_config(odp_pktio_t hdl, const odp_pktio_config_t *config) /* Check config for invalid values */ if (config->pktin.all_bits & ~capa.config.pktin.all_bits) { - ODP_ERR("Unsupported input configuration option\n"); + _ODP_ERR("Unsupported input configuration option\n"); return -1; } if (config->pktout.all_bits & ~capa.config.pktout.all_bits) { - ODP_ERR("Unsupported output configuration option\n"); + _ODP_ERR("Unsupported output configuration option\n"); return -1; } if (config->enable_loop && !capa.config.enable_loop) { - ODP_ERR("Loopback mode not supported\n"); + _ODP_ERR("Loopback mode not supported\n"); return -1; } lock_entry(entry); if (entry->state == PKTIO_STATE_STARTED) { unlock_entry(entry); - ODP_DBG("pktio %s: not stopped\n", entry->name); + _ODP_DBG("pktio %s: not stopped\n", entry->name); return -1; } @@ -619,7 +617,7 @@ int odp_pktio_config(odp_pktio_t hdl, const odp_pktio_config_t *config) if (entry->enabled.tx_compl) if (configure_tx_event_compl(entry)) { unlock_entry(entry); - ODP_ERR("Unable to configure Tx event completion\n"); + _ODP_ERR("Unable to configure Tx event completion\n"); return -1; } @@ -641,14 +639,14 @@ int odp_pktio_start(odp_pktio_t hdl) entry = get_pktio_entry(hdl); if (!entry) { - ODP_ERR("Bad handle\n"); + _ODP_ERR("Bad handle\n"); return -1; } lock_entry(entry); if (entry->state == PKTIO_STATE_STARTED) { unlock_entry(entry); - ODP_ERR("Already started\n"); + _ODP_ERR("Already started\n"); return -1; } entry->parse_layer = pktio_cls_enabled(entry) ? @@ -674,7 +672,7 @@ int odp_pktio_start(odp_pktio_t hdl) odpq[i] = entry->in_queue[i].queue; if (entry->in_queue[i].queue == ODP_QUEUE_INVALID) { - ODP_ERR("No input queue\n"); + _ODP_ERR("No input queue\n"); return -1; } } @@ -682,12 +680,12 @@ int odp_pktio_start(odp_pktio_t hdl) _odp_sched_fn->pktio_start(odp_pktio_index(hdl), num, index, odpq); } - ODP_DBG("interface: %s, input queues: %u, output queues: %u\n", - entry->name, entry->num_in_queue, entry->num_out_queue); + _ODP_DBG("interface: %s, input queues: %u, output queues: %u\n", + entry->name, entry->num_in_queue, entry->num_out_queue); if (_ODP_PCAPNG) { if (_odp_pcapng_start(entry)) - ODP_ERR("pcapng start failed, won't capture\n"); + _ODP_ERR("pcapng start failed, won't capture\n"); } return res; @@ -699,7 +697,7 @@ static int _pktio_stop(pktio_entry_t *entry) odp_pktin_mode_t mode = entry->param.in_mode; if (entry->state != PKTIO_STATE_STARTED) { - ODP_ERR("Not started\n"); + _ODP_ERR("Not started\n"); return -1; } @@ -727,7 +725,7 @@ int odp_pktio_stop(odp_pktio_t hdl) entry = get_pktio_entry(hdl); if (!entry) { - ODP_ERR("Bad handle\n"); + _ODP_ERR("Bad handle\n"); return -1; } @@ -735,7 +733,7 @@ int odp_pktio_stop(odp_pktio_t hdl) res = _pktio_stop(entry); unlock_entry(entry); - ODP_DBG("interface: %s\n", entry->name); + _ODP_DBG("interface: %s\n", entry->name); return res; } @@ -999,7 +997,7 @@ static inline int packet_vector_send(odp_pktout_queue_t pktout_queue, odp_event_ int num, sent; num = odp_packet_vector_tbl(pktv, &pkt_tbl); - ODP_ASSERT(num > 0); + _ODP_ASSERT(num > 0); sent = odp_pktout_send(pktout_queue, pkt_tbl, num); /* Return success if any packets were sent. Free the possible remaining @@ -1010,7 +1008,7 @@ static inline int packet_vector_send(odp_pktout_queue_t pktout_queue, odp_event_ pktio_entry_t *entry = get_pktio_entry(pktout_queue.pktio); int discards = num - sent; - ODP_ASSERT(entry != NULL); + _ODP_ASSERT(entry != NULL); odp_atomic_add_u64(&entry->stats_extra.out_discards, discards); @@ -1102,7 +1100,7 @@ static _odp_event_hdr_t *pktin_dequeue(odp_queue_t queue) int pktin_index = pktin_queue.index; pktio_entry_t *entry = get_pktio_entry(pktio); - ODP_ASSERT(entry != NULL); + _ODP_ASSERT(entry != NULL); if (_odp_queue_fn->orig_deq_multi(queue, &event_hdr, 1) == 1) return event_hdr; @@ -1123,8 +1121,7 @@ static _odp_event_hdr_t *pktin_dequeue(odp_queue_t queue) if (odp_unlikely(num_enq < 0)) num_enq = 0; - ODP_DBG("Interface %s dropped %i packets\n", - entry->name, num - num_enq); + _ODP_DBG("Interface %s dropped %i packets\n", entry->name, num - num_enq); _odp_event_free_multi(&hdr_tbl[num_enq + 1], num - num_enq); } } @@ -1144,11 +1141,11 @@ static int pktin_deq_multi(odp_queue_t queue, _odp_event_hdr_t *event_hdr[], int pktin_index = pktin_queue.index; pktio_entry_t *entry = get_pktio_entry(pktio); - ODP_ASSERT(entry != NULL); + _ODP_ASSERT(entry != NULL); nbr = _odp_queue_fn->orig_deq_multi(queue, event_hdr, num); if (odp_unlikely(nbr > num)) - ODP_ABORT("queue_deq_multi req: %d, returned %d\n", num, nbr); + _ODP_ABORT("queue_deq_multi req: %d, returned %d\n", num, nbr); /** queue already has number of requested buffers, * do not do receive in that case. @@ -1177,8 +1174,8 @@ static int pktin_deq_multi(odp_queue_t queue, _odp_event_hdr_t *event_hdr[], if (odp_unlikely(num_enq < 0)) num_enq = 0; - ODP_DBG("Interface %s dropped %i packets\n", - entry->name, j - num_enq); + _ODP_DBG("Interface %s dropped %i packets\n", + entry->name, j - num_enq); _odp_event_free_multi(&event_hdr[num_enq], j - num_enq); } } @@ -1197,7 +1194,7 @@ int _odp_sched_cb_pktin_poll(int pktio_index, int pktin_index, state == PKTIO_STATE_STOP_PENDING) return -1; - ODP_DBG("Interface %s not started\n", entry->name); + _ODP_DBG("Interface %s not started\n", entry->name); return 0; } @@ -1216,7 +1213,7 @@ void _odp_sched_cb_pktio_stop_finalize(int pktio_index) if (state != PKTIO_STATE_STOP_PENDING && state != PKTIO_STATE_CLOSE_PENDING) { unlock_entry(entry); - ODP_ERR("Not in a pending state %i\n", state); + _ODP_ERR("Not in a pending state %i\n", state); return; } @@ -1235,7 +1232,7 @@ static inline uint32_t pktio_maxlen(odp_pktio_t hdl) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return 0; } @@ -1243,7 +1240,7 @@ static inline uint32_t pktio_maxlen(odp_pktio_t hdl) if (odp_unlikely(is_free(entry))) { unlock_entry(entry); - ODP_DBG("already freed pktio\n"); + _ODP_DBG("already freed pktio\n"); return 0; } @@ -1273,51 +1270,51 @@ int odp_pktio_maxlen_set(odp_pktio_t hdl, uint32_t maxlen_input, entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_ERR("Pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); + _ODP_ERR("Pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return -1; } ret = odp_pktio_capability(hdl, &capa); if (ret) { - ODP_ERR("Reading pktio capability failed\n"); + _ODP_ERR("Reading pktio capability failed\n"); goto fail; } lock_entry(entry); if (odp_unlikely(is_free(entry))) { - ODP_ERR("Pktio already freed\n"); + _ODP_ERR("Pktio already freed\n"); ret = -1; goto fail; } if (entry->state == PKTIO_STATE_STARTED) { - ODP_ERR("Pktio not stopped\n"); + _ODP_ERR("Pktio not stopped\n"); ret = -1; goto fail; } if (capa.set_op.op.maxlen == 0) { - ODP_ERR("Setting maximum frame length not supported\n"); + _ODP_ERR("Setting maximum frame length not supported\n"); ret = -1; goto fail; } if (capa.maxlen.equal && (maxlen_input != maxlen_output)) { - ODP_ERR("Max input and output lengths don't match\n"); + _ODP_ERR("Max input and output lengths don't match\n"); ret = -1; goto fail; } if (maxlen_input < capa.maxlen.min_input || maxlen_input > capa.maxlen.max_input) { - ODP_ERR("Invalid max input length value: %" PRIu32 "\n", maxlen_input); + _ODP_ERR("Invalid max input length value: %" PRIu32 "\n", maxlen_input); ret = -1; goto fail; } if (maxlen_output < capa.maxlen.min_output || maxlen_output > capa.maxlen.max_output) { - ODP_ERR("Invalid max output length value: %" PRIu32 "\n", maxlen_output); + _ODP_ERR("Invalid max output length value: %" PRIu32 "\n", maxlen_output); ret = -1; goto fail; } @@ -1337,7 +1334,7 @@ int odp_pktio_promisc_mode_set(odp_pktio_t hdl, odp_bool_t enable) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return -1; } @@ -1345,7 +1342,7 @@ int odp_pktio_promisc_mode_set(odp_pktio_t hdl, odp_bool_t enable) if (odp_unlikely(is_free(entry))) { unlock_entry(entry); - ODP_DBG("already freed pktio\n"); + _ODP_DBG("already freed pktio\n"); return -1; } if (entry->state == PKTIO_STATE_STARTED) { @@ -1367,7 +1364,7 @@ int odp_pktio_promisc_mode(odp_pktio_t hdl) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return -1; } @@ -1375,7 +1372,7 @@ int odp_pktio_promisc_mode(odp_pktio_t hdl) if (odp_unlikely(is_free(entry))) { unlock_entry(entry); - ODP_DBG("already freed pktio\n"); + _ODP_DBG("already freed pktio\n"); return -1; } @@ -1398,7 +1395,7 @@ int odp_pktio_mac_addr(odp_pktio_t hdl, void *mac_addr, int addr_size) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return -1; } @@ -1406,14 +1403,14 @@ int odp_pktio_mac_addr(odp_pktio_t hdl, void *mac_addr, int addr_size) if (odp_unlikely(is_free(entry))) { unlock_entry(entry); - ODP_DBG("already freed pktio\n"); + _ODP_DBG("already freed pktio\n"); return -1; } if (entry->ops->mac_get) { ret = entry->ops->mac_get(entry, mac_addr); } else { - ODP_DBG("pktio does not support mac addr get\n"); + _ODP_DBG("pktio does not support mac addr get\n"); ret = -1; } unlock_entry(entry); @@ -1433,7 +1430,7 @@ int odp_pktio_mac_addr_set(odp_pktio_t hdl, const void *mac_addr, int addr_size) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return -1; } @@ -1441,7 +1438,7 @@ int odp_pktio_mac_addr_set(odp_pktio_t hdl, const void *mac_addr, int addr_size) if (odp_unlikely(is_free(entry))) { unlock_entry(entry); - ODP_DBG("already freed pktio\n"); + _ODP_DBG("already freed pktio\n"); return -1; } @@ -1464,7 +1461,7 @@ odp_pktio_link_status_t odp_pktio_link_status(odp_pktio_t hdl) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return ODP_PKTIO_LINK_STATUS_UNKNOWN; } @@ -1472,7 +1469,7 @@ odp_pktio_link_status_t odp_pktio_link_status(odp_pktio_t hdl) if (odp_unlikely(is_free(entry))) { unlock_entry(entry); - ODP_DBG("already freed pktio\n"); + _ODP_DBG("already freed pktio\n"); return ODP_PKTIO_LINK_STATUS_UNKNOWN; } @@ -1521,7 +1518,7 @@ int odp_pktio_info(odp_pktio_t hdl, odp_pktio_info_t *info) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return -1; } @@ -1541,7 +1538,7 @@ int odp_pktio_link_info(odp_pktio_t hdl, odp_pktio_link_info_t *info) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return -1; } @@ -1557,7 +1554,7 @@ uint64_t odp_pktio_ts_res(odp_pktio_t hdl) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); + _ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return 0; } @@ -1574,7 +1571,7 @@ odp_time_t odp_pktio_ts_from_ns(odp_pktio_t hdl, uint64_t ns) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); + _ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return ODP_TIME_NULL; } @@ -1591,7 +1588,7 @@ odp_time_t odp_pktio_time(odp_pktio_t hdl, odp_time_t *global_ts) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); + _ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return ODP_TIME_NULL; } @@ -1630,7 +1627,7 @@ void odp_pktio_print(odp_pktio_t hdl) entry = get_pktio_entry(hdl); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return; } @@ -1679,12 +1676,12 @@ void odp_pktio_print(odp_pktio_t hdl) str[len] = '\0'; - ODP_PRINT("\n%s", str); + _ODP_PRINT("\n%s", str); if (entry->ops->print) entry->ops->print(entry); - ODP_PRINT("\n"); + _ODP_PRINT("\n"); } int _odp_pktio_term_global(void) @@ -1708,35 +1705,32 @@ int _odp_pktio_term_global(void) if (pktio_entry->state == PKTIO_STATE_STARTED) { ret = _pktio_stop(pktio_entry); if (ret) - ODP_ABORT("unable to stop pktio %s\n", - pktio_entry->name); + _ODP_ABORT("unable to stop pktio %s\n", pktio_entry->name); } if (pktio_entry->state != PKTIO_STATE_CLOSE_PENDING) ret = _pktio_close(pktio_entry); if (ret) - ODP_ABORT("unable to close pktio %s\n", - pktio_entry->name); + _ODP_ABORT("unable to close pktio %s\n", pktio_entry->name); unlock_entry(pktio_entry); } for (pktio_if = 0; _odp_pktio_if_ops[pktio_if]; ++pktio_if) { if (_odp_pktio_if_ops[pktio_if]->term) if (_odp_pktio_if_ops[pktio_if]->term()) - ODP_ABORT("failed to terminate pktio type %d", - pktio_if); + _ODP_ABORT("failed to terminate pktio type %d", pktio_if); } if (_ODP_PCAPNG) { ret = _odp_pcapng_term_global(); if (ret) - ODP_ERR("Failed to terminate pcapng\n"); + _ODP_ERR("Failed to terminate pcapng\n"); } shm = pktio_global->shm; ret = odp_shm_free(shm); if (ret != 0) - ODP_ERR("shm free failed\n"); + _ODP_ERR("shm free failed\n"); return ret; } @@ -1759,7 +1753,7 @@ int odp_pktio_capability(odp_pktio_t pktio, odp_pktio_capability_t *capa) entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -1772,7 +1766,7 @@ int odp_pktio_capability(odp_pktio_t pktio, odp_pktio_capability_t *capa) uint32_t mtu = pktio_maxlen(pktio); if (mtu == 0) { - ODP_DBG("MTU query failed: %s\n", entry->name); + _ODP_DBG("MTU query failed: %s\n", entry->name); return -1; } @@ -1836,7 +1830,7 @@ int odp_pktio_stats(odp_pktio_t pktio, entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -1844,7 +1838,7 @@ int odp_pktio_stats(odp_pktio_t pktio, if (odp_unlikely(is_free(entry))) { unlock_entry(entry); - ODP_DBG("already freed pktio\n"); + _ODP_DBG("already freed pktio\n"); return -1; } @@ -1867,7 +1861,7 @@ int odp_pktio_stats_reset(odp_pktio_t pktio) entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -1875,7 +1869,7 @@ int odp_pktio_stats_reset(odp_pktio_t pktio) if (odp_unlikely(is_free(entry))) { unlock_entry(entry); - ODP_DBG("already freed pktio\n"); + _ODP_DBG("already freed pktio\n"); return -1; } @@ -1898,7 +1892,7 @@ int odp_pktin_queue_stats(odp_pktin_queue_t queue, entry = get_pktio_entry(queue.pktio); if (entry == NULL) { - ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)queue.pktio); + _ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)queue.pktio); return -1; } @@ -1906,14 +1900,14 @@ int odp_pktin_queue_stats(odp_pktin_queue_t queue, if (odp_unlikely(is_free(entry))) { unlock_entry(entry); - ODP_ERR("pktio entry already freed\n"); + _ODP_ERR("pktio entry already freed\n"); return -1; } mode = entry->param.in_mode; if (odp_unlikely(mode != ODP_PKTIN_MODE_DIRECT)) { unlock_entry(entry); - ODP_ERR("invalid packet input mode: %d\n", mode); + _ODP_ERR("invalid packet input mode: %d\n", mode); return -1; } @@ -1935,7 +1929,7 @@ int odp_pktin_event_queue_stats(odp_pktio_t pktio, odp_queue_t queue, entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); + _ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -1943,14 +1937,14 @@ int odp_pktin_event_queue_stats(odp_pktio_t pktio, odp_queue_t queue, if (odp_unlikely(is_free(entry))) { unlock_entry(entry); - ODP_ERR("pktio entry already freed\n"); + _ODP_ERR("pktio entry already freed\n"); return -1; } mode = entry->param.in_mode; if (odp_unlikely(mode != ODP_PKTIN_MODE_SCHED && mode != ODP_PKTIN_MODE_QUEUE)) { unlock_entry(entry); - ODP_ERR("invalid packet input mode: %d\n", mode); + _ODP_ERR("invalid packet input mode: %d\n", mode); return -1; } @@ -1973,7 +1967,7 @@ int odp_pktout_queue_stats(odp_pktout_queue_t queue, entry = get_pktio_entry(queue.pktio); if (entry == NULL) { - ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)queue.pktio); + _ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)queue.pktio); return -1; } @@ -1981,14 +1975,14 @@ int odp_pktout_queue_stats(odp_pktout_queue_t queue, if (odp_unlikely(is_free(entry))) { unlock_entry(entry); - ODP_ERR("pktio entry already freed\n"); + _ODP_ERR("pktio entry already freed\n"); return -1; } mode = entry->param.out_mode; if (odp_unlikely(mode != ODP_PKTOUT_MODE_DIRECT)) { unlock_entry(entry); - ODP_ERR("invalid packet output mode: %d\n", mode); + _ODP_ERR("invalid packet output mode: %d\n", mode); return -1; } @@ -2010,7 +2004,7 @@ int odp_pktout_event_queue_stats(odp_pktio_t pktio, odp_queue_t queue, entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); + _ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -2018,14 +2012,14 @@ int odp_pktout_event_queue_stats(odp_pktio_t pktio, odp_queue_t queue, if (odp_unlikely(is_free(entry))) { unlock_entry(entry); - ODP_ERR("pktio entry already freed\n"); + _ODP_ERR("pktio entry already freed\n"); return -1; } mode = entry->param.out_mode; if (odp_unlikely(mode != ODP_PKTOUT_MODE_QUEUE)) { unlock_entry(entry); - ODP_ERR("invalid packet output mode: %d\n", mode); + _ODP_ERR("invalid packet output mode: %d\n", mode); return -1; } @@ -2047,7 +2041,7 @@ int odp_pktio_extra_stat_info(odp_pktio_t pktio, entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); + _ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -2055,7 +2049,7 @@ int odp_pktio_extra_stat_info(odp_pktio_t pktio, if (odp_unlikely(is_free(entry))) { unlock_entry(entry); - ODP_ERR("already freed pktio\n"); + _ODP_ERR("already freed pktio\n"); return -1; } @@ -2074,7 +2068,7 @@ int odp_pktio_extra_stats(odp_pktio_t pktio, uint64_t stats[], int num) entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); + _ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -2082,7 +2076,7 @@ int odp_pktio_extra_stats(odp_pktio_t pktio, uint64_t stats[], int num) if (odp_unlikely(is_free(entry))) { unlock_entry(entry); - ODP_ERR("already freed pktio\n"); + _ODP_ERR("already freed pktio\n"); return -1; } @@ -2101,7 +2095,7 @@ int odp_pktio_extra_stat_counter(odp_pktio_t pktio, uint32_t id, uint64_t *stat) entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); + _ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -2109,7 +2103,7 @@ int odp_pktio_extra_stat_counter(odp_pktio_t pktio, uint32_t id, uint64_t *stat) if (odp_unlikely(is_free(entry))) { unlock_entry(entry); - ODP_ERR("already freed pktio\n"); + _ODP_ERR("already freed pktio\n"); return -1; } @@ -2134,7 +2128,7 @@ void odp_pktio_extra_stats_print(odp_pktio_t pktio) return; if (num_info != num_stats) { - ODP_ERR("extra statistics info counts not matching\n"); + _ODP_ERR("extra statistics info counts not matching\n"); return; } @@ -2150,14 +2144,14 @@ void odp_pktio_extra_stats_print(odp_pktio_t pktio) return; if (num_info != num_stats) { - ODP_ERR("extra statistics info counts not matching\n"); + _ODP_ERR("extra statistics info counts not matching\n"); return; } - ODP_PRINT("Pktio extra statistics\n----------------------\n"); + _ODP_PRINT("Pktio extra statistics\n----------------------\n"); for (i = 0; i < num_stats; i++) - ODP_PRINT(" %s=%" PRIu64 "\n", stats_info[i].name, extra_stats[i]); - ODP_PRINT("\n"); + _ODP_PRINT(" %s=%" PRIu64 "\n", stats_info[i].name, extra_stats[i]); + _ODP_PRINT("\n"); } int odp_pktin_queue_config(odp_pktio_t pktio, @@ -2178,12 +2172,12 @@ int odp_pktin_queue_config(odp_pktio_t pktio, entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } if (entry->state == PKTIO_STATE_STARTED) { - ODP_DBG("pktio %s: not stopped\n", entry->name); + _ODP_DBG("pktio %s: not stopped\n", entry->name); return -1; } @@ -2194,7 +2188,7 @@ int odp_pktin_queue_config(odp_pktio_t pktio, return 0; if (!param->classifier_enable && param->num_queues == 0) { - ODP_DBG("invalid num_queues for operation mode\n"); + _ODP_DBG("invalid num_queues for operation mode\n"); return -1; } @@ -2202,15 +2196,14 @@ int odp_pktin_queue_config(odp_pktio_t pktio, rc = odp_pktio_capability(pktio, &capa); if (rc) { - ODP_DBG("pktio %s: unable to read capabilities\n", - entry->name); + _ODP_DBG("pktio %s: unable to read capabilities\n", entry->name); return -1; } entry->enabled.cls = !!param->classifier_enable; if (num_queues > capa.max_input_queues) { - ODP_DBG("pktio %s: too many input queues\n", entry->name); + _ODP_DBG("pktio %s: too many input queues\n", entry->name); return -1; } @@ -2220,35 +2213,35 @@ int odp_pktin_queue_config(odp_pktio_t pktio, odp_pool_info_t pool_info; if (mode == ODP_PKTIN_MODE_DIRECT) { - ODP_ERR("packet vectors not supported with ODP_PKTIN_MODE_DIRECT\n"); + _ODP_ERR("packet vectors not supported with ODP_PKTIN_MODE_DIRECT\n"); return -1; } if (param->vector.max_size < capa.vector.min_size) { - ODP_ERR("vector.max_size too small %" PRIu32 "\n", - param->vector.max_size); + _ODP_ERR("vector.max_size too small %" PRIu32 "\n", + param->vector.max_size); return -1; } if (param->vector.max_size > capa.vector.max_size) { - ODP_ERR("vector.max_size too large %" PRIu32 "\n", - param->vector.max_size); + _ODP_ERR("vector.max_size too large %" PRIu32 "\n", + param->vector.max_size); return -1; } if (param->vector.max_tmo_ns > capa.vector.max_tmo_ns) { - ODP_ERR("vector.max_tmo_ns too large %" PRIu64 "\n", - param->vector.max_tmo_ns); + _ODP_ERR("vector.max_tmo_ns too large %" PRIu64 "\n", + param->vector.max_tmo_ns); return -1; } if (pool == ODP_POOL_INVALID || odp_pool_info(pool, &pool_info)) { - ODP_ERR("invalid packet vector pool\n"); + _ODP_ERR("invalid packet vector pool\n"); return -1; } if (pool_info.params.type != ODP_POOL_VECTOR) { - ODP_ERR("wrong pool type\n"); + _ODP_ERR("wrong pool type\n"); return -1; } if (param->vector.max_size > pool_info.params.vector.max_size) { - ODP_ERR("vector.max_size larger than pool max vector size\n"); + _ODP_ERR("vector.max_size larger than pool max vector size\n"); return -1; } } @@ -2289,8 +2282,7 @@ int odp_pktin_queue_config(odp_pktio_t pktio, queue = odp_queue_create(name, &queue_param); if (queue == ODP_QUEUE_INVALID) { - ODP_DBG("pktio %s: event queue create failed\n", - entry->name); + _ODP_DBG("pktio %s: event queue create failed\n", entry->name); destroy_in_queues(entry, i + 1); return -1; } @@ -2337,7 +2329,7 @@ int _odp_pktio_pktout_tm_config(odp_pktio_t pktio_hdl, entry = get_pktio_entry(pktio_hdl); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio_hdl); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio_hdl); return -1; } @@ -2357,7 +2349,7 @@ int _odp_pktio_pktout_tm_config(odp_pktio_t pktio_hdl, pktio_started = true; rc = odp_pktio_stop(pktio_hdl); if (rc) { - ODP_ERR("Unable to stop pktio, rc=%d\n", rc); + _ODP_ERR("Unable to stop pktio, rc=%d\n", rc); return rc; } } @@ -2380,7 +2372,7 @@ int _odp_pktio_pktout_tm_config(odp_pktio_t pktio_hdl, if (entry->ops->output_queues_config) { rc = entry->ops->output_queues_config(entry, ¶m); if (rc) - ODP_ERR("Unable to setup output queues, rc=%d\n", rc); + _ODP_ERR("Unable to setup output queues, rc=%d\n", rc); } /* Return pktout queue on success */ @@ -2410,12 +2402,12 @@ int odp_pktout_queue_config(odp_pktio_t pktio, entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); + _ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } if (entry->state == PKTIO_STATE_STARTED) { - ODP_ERR("pktio %s: not stopped\n", entry->name); + _ODP_ERR("pktio %s: not stopped\n", entry->name); return -1; } @@ -2427,26 +2419,25 @@ int odp_pktout_queue_config(odp_pktio_t pktio, return 0; if (mode != ODP_PKTOUT_MODE_DIRECT && mode != ODP_PKTOUT_MODE_QUEUE) { - ODP_ERR("pktio %s: bad packet output mode\n", entry->name); + _ODP_ERR("pktio %s: bad packet output mode\n", entry->name); return -1; } num_queues = param->num_queues; if (num_queues == 0) { - ODP_ERR("pktio %s: zero output queues\n", entry->name); + _ODP_ERR("pktio %s: zero output queues\n", entry->name); return -1; } rc = odp_pktio_capability(pktio, &capa); if (rc) { - ODP_ERR("pktio %s: unable to read capabilities\n", - entry->name); + _ODP_ERR("pktio %s: unable to read capabilities\n", entry->name); return -1; } if (num_queues > capa.max_output_queues) { - ODP_ERR("pktio %s: too many output queues\n", entry->name); + _ODP_ERR("pktio %s: too many output queues\n", entry->name); return -1; } @@ -2458,16 +2449,16 @@ int odp_pktout_queue_config(odp_pktio_t pktio, continue; if (capa.max_output_queue_size == 0) { - ODP_ERR("pktio %s: configuring output queue size not supported\n", - entry->name); + _ODP_ERR("pktio %s: configuring output queue size not supported\n", + entry->name); return -1; } if (queue_size < capa.min_output_queue_size) { - ODP_ERR("pktio %s: output queue size too small\n", entry->name); + _ODP_ERR("pktio %s: output queue size too small\n", entry->name); return -1; } if (queue_size > capa.max_output_queue_size) { - ODP_ERR("pktio %s: output queue size too large\n", entry->name); + _ODP_ERR("pktio %s: output queue size too large\n", entry->name); return -1; } } @@ -2504,8 +2495,7 @@ int odp_pktout_queue_config(odp_pktio_t pktio, queue = odp_queue_create(name, &queue_param); if (queue == ODP_QUEUE_INVALID) { - ODP_ERR("pktout %s: event queue create failed\n", - entry->name); + _ODP_ERR("pktout %s: event queue create failed\n", entry->name); destroy_out_queues(entry, i + 1); return -1; } @@ -2538,12 +2528,12 @@ int odp_pktin_event_queue(odp_pktio_t pktio, odp_queue_t queues[], int num) entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } if (num < 0) { - ODP_DBG("Bad param: num %i\n", num); + _ODP_DBG("Bad param: num %i\n", num); return -1; } @@ -2578,12 +2568,12 @@ int odp_pktin_queue(odp_pktio_t pktio, odp_pktin_queue_t queues[], int num) entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } if (num < 0) { - ODP_DBG("Bad param: num %i\n", num); + _ODP_DBG("Bad param: num %i\n", num); return -1; } @@ -2617,7 +2607,7 @@ int odp_pktout_event_queue(odp_pktio_t pktio, odp_queue_t queues[], int num) entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -2648,7 +2638,7 @@ int odp_pktout_queue(odp_pktio_t pktio, odp_pktout_queue_t queues[], int num) entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -2685,7 +2675,7 @@ int odp_pktin_recv(odp_pktin_queue_t queue, odp_packet_t packets[], int num) entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -2714,7 +2704,7 @@ int odp_pktin_recv_tmo(odp_pktin_queue_t queue, odp_packet_t packets[], int num, entry = get_pktio_entry(queue.pktio); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)queue.pktio); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)queue.pktio); return -1; } @@ -2882,7 +2872,7 @@ static void send_tx_compl_event(odp_buffer_t buf, const void *user_ptr, odp_queu _odp_event_type_set(ev, ODP_EVENT_PACKET_TX_COMPL); if (odp_unlikely(odp_queue_enq(queue, ev))) { - ODP_ERR("Failed to enqueue Tx completion event\n"); + _ODP_ERR("Failed to enqueue Tx completion event\n"); odp_event_free(ev); } } @@ -2912,7 +2902,7 @@ int odp_pktout_send(odp_pktout_queue_t queue, const odp_packet_t packets[], entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)pktio); return -1; } @@ -2961,7 +2951,7 @@ int odp_pktout_ts_read(odp_pktio_t hdl, odp_time_t *ts) entry = get_pktio_entry(hdl); if (odp_unlikely(entry == NULL)) { - ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); + _ODP_ERR("pktio entry %" PRIuPTR " does not exist\n", (uintptr_t)hdl); return -1; } @@ -2992,14 +2982,14 @@ odp_lso_profile_t odp_lso_profile_create(odp_pktio_t pktio, const odp_lso_profil /* Currently only IPv4 and custom implemented */ if (param->lso_proto != ODP_LSO_PROTO_IPV4 && param->lso_proto != ODP_LSO_PROTO_CUSTOM) { - ODP_ERR("Protocol not supported\n"); + _ODP_ERR("Protocol not supported\n"); return ODP_LSO_PROFILE_INVALID; } if (param->lso_proto == ODP_LSO_PROTO_CUSTOM) { num_custom = param->custom.num_custom; if (num_custom > ODP_LSO_MAX_CUSTOM) { - ODP_ERR("Too many custom fields\n"); + _ODP_ERR("Too many custom fields\n"); return ODP_LSO_PROFILE_INVALID; } @@ -3009,18 +2999,18 @@ odp_lso_profile_t odp_lso_profile_create(odp_pktio_t pktio, const odp_lso_profil size = param->custom.field[i].size; if (offset > PKTIO_LSO_MAX_PAYLOAD_OFFSET) { - ODP_ERR("Too large custom field offset %u\n", offset); + _ODP_ERR("Too large custom field offset %u\n", offset); return ODP_LSO_PROFILE_INVALID; } /* Currently only segment number supported */ if (mod_op != ODP_LSO_ADD_SEGMENT_NUM) { - ODP_ERR("Custom modify operation %u not supported\n", mod_op); + _ODP_ERR("Custom modify operation %u not supported\n", mod_op); return ODP_LSO_PROFILE_INVALID; } if (size != 1 && size != 2 && size != 4 && size != 8) { - ODP_ERR("Bad custom field size %u\n", size); + _ODP_ERR("Bad custom field size %u\n", size); return ODP_LSO_PROFILE_INVALID; } } @@ -3030,7 +3020,7 @@ odp_lso_profile_t odp_lso_profile_create(odp_pktio_t pktio, const odp_lso_profil if (pktio_global->num_lso_profiles >= PKTIO_LSO_PROFILES) { odp_spinlock_unlock(&pktio_global->lock); - ODP_ERR("All LSO profiles used already: %u\n", PKTIO_LSO_PROFILES); + _ODP_ERR("All LSO profiles used already: %u\n", PKTIO_LSO_PROFILES); return ODP_LSO_PROFILE_INVALID; } @@ -3046,7 +3036,7 @@ odp_lso_profile_t odp_lso_profile_create(odp_pktio_t pktio, const odp_lso_profil odp_spinlock_unlock(&pktio_global->lock); if (lso_prof == NULL) { - ODP_ERR("Did not find free LSO profile\n"); + _ODP_ERR("Did not find free LSO profile\n"); return ODP_LSO_PROFILE_INVALID; } @@ -3071,7 +3061,7 @@ int odp_lso_profile_destroy(odp_lso_profile_t lso_profile) lso_profile_t *lso_prof = lso_profile_ptr(lso_profile); if (lso_profile == ODP_LSO_PROFILE_INVALID || lso_prof->used == 0) { - ODP_ERR("Bad handle\n"); + _ODP_ERR("Bad handle\n"); return -1; } @@ -3090,22 +3080,22 @@ int odp_packet_lso_request(odp_packet_t pkt, const odp_packet_lso_opt_t *lso_opt uint32_t payload_offset = lso_opt->payload_offset; if (odp_unlikely(lso_opt->lso_profile == ODP_LSO_PROFILE_INVALID || lso_prof->used == 0)) { - ODP_ERR("Bad LSO profile handle\n"); + _ODP_ERR("Bad LSO profile handle\n"); return -1; } if (odp_unlikely(payload_offset > PKTIO_LSO_MAX_PAYLOAD_OFFSET)) { - ODP_ERR("Too large LSO payload offset\n"); + _ODP_ERR("Too large LSO payload offset\n"); return -1; } if (odp_unlikely(payload_offset > packet_len(pkt_hdr))) { - ODP_ERR("LSO payload offset larger than packet data length\n"); + _ODP_ERR("LSO payload offset larger than packet data length\n"); return -1; } if (odp_packet_payload_offset_set(pkt, payload_offset)) { - ODP_ERR("Payload offset set failed\n"); + _ODP_ERR("Payload offset set failed\n"); return -1; } @@ -3169,7 +3159,7 @@ static int lso_update_custom(lso_profile_t *lso_prof, odp_packet_t pkt, int segn ptr = &u8; if (odp_packet_copy_to_mem(pkt, offset, size, ptr)) { - ODP_ERR("Read from packet failed at offset %u\n", offset); + _ODP_ERR("Read from packet failed at offset %u\n", offset); return -1; } @@ -3185,7 +3175,7 @@ static int lso_update_custom(lso_profile_t *lso_prof, odp_packet_t pkt, int segn } if (odp_packet_copy_from_mem(pkt, offset, size, ptr)) { - ODP_ERR("Write to packet failed at offset %u\n", offset); + _ODP_ERR("Write to packet failed at offset %u\n", offset); return -1; } } @@ -3205,12 +3195,12 @@ int _odp_lso_num_packets(odp_packet_t packet, const odp_packet_lso_opt_t *lso_op uint32_t pkt_payload = pkt_len - hdr_len; if (odp_unlikely(hdr_len > PKTIO_LSO_MAX_PAYLOAD_OFFSET)) { - ODP_ERR("Too large LSO payload offset\n"); + _ODP_ERR("Too large LSO payload offset\n"); return -1; } if (odp_unlikely(hdr_len > pkt_len)) { - ODP_ERR("LSO payload offset larger than packet data length\n"); + _ODP_ERR("LSO payload offset larger than packet data length\n"); return -1; } @@ -3227,12 +3217,12 @@ int _odp_lso_num_packets(odp_packet_t packet, const odp_packet_lso_opt_t *lso_op iphdr_len = hdr_len - l3_offset; if (l3_offset == ODP_PACKET_OFFSET_INVALID) { - ODP_ERR("Invalid L3 offset\n"); + _ODP_ERR("Invalid L3 offset\n"); return -1; } if (hdr_len < l3_offset || iphdr_len < _ODP_IPV4HDR_LEN) { - ODP_ERR("Bad payload or L3 offset\n"); + _ODP_ERR("Bad payload or L3 offset\n"); return -1; } @@ -3247,8 +3237,8 @@ int _odp_lso_num_packets(odp_packet_t packet, const odp_packet_lso_opt_t *lso_op num_pkt++; if (num_pkt > PKTIO_LSO_MAX_SEGMENTS) { - ODP_ERR("Too many LSO segments %i. Maximum is %i\n", num_pkt, - PKTIO_LSO_MAX_SEGMENTS); + _ODP_ERR("Too many LSO segments %i. Maximum is %i\n", num_pkt, + PKTIO_LSO_MAX_SEGMENTS); return -1; } @@ -3278,7 +3268,7 @@ int _odp_lso_create_packets(odp_packet_t packet, const odp_packet_lso_opt_t *lso num = odp_packet_alloc_multi(pool, pkt_len, pkt_out, num_full); if (odp_unlikely(num < num_full)) { - ODP_DBG("Alloc failed %i\n", num); + _ODP_DBG("Alloc failed %i\n", num); if (num > 0) { num_free = num; goto error; @@ -3288,7 +3278,7 @@ int _odp_lso_create_packets(odp_packet_t packet, const odp_packet_lso_opt_t *lso if (left_over_len) { pkt = odp_packet_alloc(pool, hdr_len + left_over_len); if (pkt == ODP_PACKET_INVALID) { - ODP_DBG("Alloc failed\n"); + _ODP_DBG("Alloc failed\n"); num_free = num_full; goto error; } @@ -3301,7 +3291,7 @@ int _odp_lso_create_packets(odp_packet_t packet, const odp_packet_lso_opt_t *lso /* Copy headers */ for (i = 0; i < num_pkt; i++) { if (odp_packet_copy_from_pkt(pkt_out[i], 0, packet, 0, hdr_len)) { - ODP_ERR("Header copy failed\n"); + _ODP_ERR("Header copy failed\n"); goto error; } } @@ -3310,7 +3300,7 @@ int _odp_lso_create_packets(odp_packet_t packet, const odp_packet_lso_opt_t *lso for (i = 0; i < num_full; i++) { offset = hdr_len + (i * payload_len); if (odp_packet_copy_from_pkt(pkt_out[i], hdr_len, packet, offset, payload_len)) { - ODP_ERR("Payload copy failed\n"); + _ODP_ERR("Payload copy failed\n"); goto error; } } @@ -3320,7 +3310,7 @@ int _odp_lso_create_packets(odp_packet_t packet, const odp_packet_lso_opt_t *lso offset = hdr_len + (num_full * payload_len); if (odp_packet_copy_from_pkt(pkt_out[num_pkt - 1], hdr_len, packet, offset, left_over_len)){ - ODP_ERR("Payload copy failed\n"); + _ODP_ERR("Payload copy failed\n"); goto error; } } @@ -3329,13 +3319,13 @@ int _odp_lso_create_packets(odp_packet_t packet, const odp_packet_lso_opt_t *lso offset = odp_packet_l3_offset(packet); if (offset == ODP_PACKET_OFFSET_INVALID) { - ODP_ERR("Invalid L3 offset\n"); + _ODP_ERR("Invalid L3 offset\n"); goto error; } for (i = 0; i < num_pkt; i++) { if (lso_update_ipv4(pkt_out[i], i, num_pkt, offset, payload_len)) { - ODP_ERR("IPv4 header update failed. Packet %i.\n", i); + _ODP_ERR("IPv4 header update failed. Packet %i.\n", i); goto error; } } @@ -3345,7 +3335,7 @@ int _odp_lso_create_packets(odp_packet_t packet, const odp_packet_lso_opt_t *lso for (i = 0; num_custom && i < num_pkt; i++) { if (lso_update_custom(lso_prof, pkt_out[i], i)) { - ODP_ERR("Custom field update failed. Segment %i\n", i); + _ODP_ERR("Custom field update failed. Segment %i\n", i); goto error; } } @@ -3393,7 +3383,7 @@ static int pktout_send_lso(odp_pktout_queue_t queue, odp_packet_t packet, int first_free = 0; int num_free = num_pkt; - ODP_DBG("Packet send failed %i\n", ret); + _ODP_DBG("Packet send failed %i\n", ret); if (ret > 0) { first_free = ret; @@ -3419,7 +3409,7 @@ int odp_pktout_send_lso(odp_pktout_queue_t queue, const odp_packet_t packet[], i const odp_packet_lso_opt_t *opt_ptr = &lso_opt; if (odp_unlikely(num <= 0)) { - ODP_ERR("No packets\n"); + _ODP_ERR("No packets\n"); return -1; } @@ -3434,7 +3424,7 @@ int odp_pktout_send_lso(odp_pktout_queue_t queue, const odp_packet_t packet[], i odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); if (pkt_hdr->p.flags.lso == 0) { - ODP_ERR("No LSO options on packet %i\n", i); + _ODP_ERR("No LSO options on packet %i\n", i); if (i == 0) return -1; @@ -3447,7 +3437,7 @@ int odp_pktout_send_lso(odp_pktout_queue_t queue, const odp_packet_t packet[], i } if (odp_unlikely(pktout_send_lso(queue, pkt, opt_ptr))) { - ODP_DBG("LSO output failed on packet %i\n", i); + _ODP_DBG("LSO output failed on packet %i\n", i); return i; } } diff --git a/platform/linux-generic/odp_packet_vector.c b/platform/linux-generic/odp_packet_vector.c index b3edbf84b..698445181 100644 --- a/platform/linux-generic/odp_packet_vector.c +++ b/platform/linux-generic/odp_packet_vector.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2020-2021, Nokia +/* Copyright (c) 2020-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -14,6 +14,7 @@ #include <odp_debug_internal.h> #include <odp_event_vector_internal.h> #include <odp_pool_internal.h> +#include <odp_print_internal.h> #include <inttypes.h> #include <stdint.h> @@ -41,17 +42,17 @@ odp_packet_vector_t odp_packet_vector_alloc(odp_pool_t pool_hdl) odp_event_t event; pool_t *pool; - ODP_ASSERT(pool_hdl != ODP_POOL_INVALID); + _ODP_ASSERT(pool_hdl != ODP_POOL_INVALID); pool = _odp_pool_entry(pool_hdl); - ODP_ASSERT(pool->type == ODP_POOL_VECTOR); + _ODP_ASSERT(pool->type == ODP_POOL_VECTOR); event = _odp_event_alloc(pool); if (odp_unlikely(event == ODP_EVENT_INVALID)) return ODP_PACKET_VECTOR_INVALID; - ODP_ASSERT(event_vector_hdr_from_event(event)->size == 0); + _ODP_ASSERT(event_vector_hdr_from_event(event)->size == 0); return odp_packet_vector_from_event(event); } @@ -107,30 +108,32 @@ void odp_packet_vector_print(odp_packet_vector_t pktv) uint32_t i; odp_event_vector_hdr_t *pktv_hdr = _odp_packet_vector_hdr(pktv); - len += snprintf(&str[len], n - len, "Packet Vector\n"); - len += snprintf(&str[len], n - len, - " handle %p\n", (void *)pktv); - len += snprintf(&str[len], n - len, - " size %" PRIu32 "\n", pktv_hdr->size); + len += _odp_snprint(&str[len], n - len, "Packet vector info\n"); + len += _odp_snprint(&str[len], n - len, "------------------\n"); + len += _odp_snprint(&str[len], n - len, " handle 0x%" PRIx64 "\n", + odp_packet_vector_to_u64(pktv)); + len += _odp_snprint(&str[len], n - len, " size %" PRIu32 "\n", pktv_hdr->size); + len += _odp_snprint(&str[len], n - len, " flags 0x%" PRIx32 "\n", + pktv_hdr->flags.all_flags); + len += _odp_snprint(&str[len], n - len, " user area %p\n", pktv_hdr->uarea_addr); for (i = 0; i < pktv_hdr->size; i++) { odp_packet_t pkt = pktv_hdr->packet[i]; char seg_str[max_len]; int str_len; - str_len = snprintf(seg_str, max_len, - " packet %p len %" PRIu32 "\n", - (void *)pkt, odp_packet_len(pkt)); + str_len = _odp_snprint(seg_str, max_len, " packet %p len %" PRIu32 "\n", + (void *)pkt, odp_packet_len(pkt)); /* Prevent print buffer overflow */ if (n - len - str_len < 10) { - len += snprintf(&str[len], n - len, " ...\n"); + len += _odp_snprint(&str[len], n - len, " ...\n"); break; } - len += snprintf(&str[len], n - len, "%s", seg_str); + len += _odp_snprint(&str[len], n - len, "%s", seg_str); } - ODP_PRINT("%s\n", str); + _ODP_PRINT("%s\n", str); } uint64_t odp_packet_vector_to_u64(odp_packet_vector_t pktv) diff --git a/platform/linux-generic/odp_parse.c b/platform/linux-generic/odp_parse.c index 41c0fb32d..d7fdb1439 100644 --- a/platform/linux-generic/odp_parse.c +++ b/platform/linux-generic/odp_parse.c @@ -179,10 +179,13 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, * Parser helper function for IPv6 * * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data. + * + * - offset is the offset of the first byte of the data pointed to by parseptr + * - seg_end is the maximum offset that can be accessed plus one */ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, - uint32_t seg_len, + uint32_t seg_end, odp_pktin_config_opt_t opt, uint64_t *l4_part_sum) { @@ -226,7 +229,7 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += extlen; } while ((ipv6ext->next_hdr == _ODP_IPPROTO_HOPOPTS || ipv6ext->next_hdr == _ODP_IPPROTO_ROUTE) && - *offset < seg_len); + *offset < seg_end); if (*offset >= prs->l3_offset + odp_be_to_cpu_16(ipv6->payload_len)) { @@ -355,10 +358,15 @@ static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_sctphdr_t); } -/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */ +/* + * Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. + * + * - offset is the offset of the first byte of the data pointed to by parseptr + * - seg_end is the maximum offset that can be accessed plus one + */ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, uint32_t offset, - uint32_t frame_len, uint32_t seg_len, + uint32_t frame_len, uint32_t seg_end, int layer, uint16_t ethtype, uint64_t *l4_part_sum, odp_pktin_config_opt_t opt) @@ -388,7 +396,7 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, case _ODP_ETHTYPE_IPV6: prs->input_flags.ipv6 = 1; ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len, - seg_len, opt, l4_part_sum); + seg_end, opt, l4_part_sum); if (odp_likely(!prs->flags.ip_err)) prs->l4_offset = offset; else if (opt.bit.drop_ipv6_err) @@ -425,7 +433,7 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, break; case _ODP_IPPROTO_TCP: - if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len)) + if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_end)) return -1; prs->input_flags.tcp = 1; parse_tcp(prs, &parseptr, frame_len - prs->l4_offset, opt, @@ -435,7 +443,7 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, break; case _ODP_IPPROTO_UDP: - if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len)) + if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_end)) return -1; prs->input_flags.udp = 1; parse_udp(prs, &parseptr, opt, l4_part_sum); diff --git a/platform/linux-generic/odp_pcapng.c b/platform/linux-generic/odp_pcapng.c index 6f1f37ef8..d97e1ca9d 100644 --- a/platform/linux-generic/odp_pcapng.c +++ b/platform/linux-generic/odp_pcapng.c @@ -11,14 +11,16 @@ #if defined(_ODP_PCAPNG) && _ODP_PCAPNG == 1 -#include <odp_pcapng.h> -#include <odp/api/plat/packet_inlines.h> #include <odp/api/shared_memory.h> +#include <odp/api/spinlock.h> + +#include <odp/api/plat/packet_inlines.h> + +#include <odp_global_data.h> #include <odp_init_internal.h> #include <odp_macros_internal.h> #include <odp_packet_io_internal.h> -#include <odp_posix_extensions.h> -#include <odp/api/spinlock.h> +#include <odp_pcapng.h> #include <errno.h> #include <fcntl.h> @@ -110,7 +112,7 @@ int _odp_pcapng_term_global(void) int ret = 0; if (odp_shm_free(pcapng_gbl->shm)) { - ODP_ERR("shm free failed"); + _ODP_ERR("shm free failed"); ret = -1; } @@ -137,8 +139,7 @@ static void inotify_event_handle(pktio_entry_t *entry, int qidx, if (PIPE_BUF < mtu + sizeof(pcapng_enhanced_packet_block_t) + sizeof(uint32_t)) { - ODP_ERR("PIPE_BUF:%d too small. Disabling pcap\n", - PIPE_BUF); + _ODP_ERR("PIPE_BUF:%d too small. Disabling pcap\n", PIPE_BUF); entry->pcapng.state[qidx] = PCAPNG_WR_STOP; return; @@ -149,16 +150,16 @@ static void inotify_event_handle(pktio_entry_t *entry, int qidx, entry->pcapng.state[qidx] = PCAPNG_WR_STOP; } else { entry->pcapng.state[qidx] = PCAPNG_WR_PKT; - ODP_DBG("Open %s for pcap tracing\n", event->name); + _ODP_DBG("Open %s for pcap tracing\n", event->name); } } else if (event->mask & IN_CLOSE) { int fd = entry->pcapng.fd[qidx]; pcapng_drain_fifo(fd); entry->pcapng.state[qidx] = PCAPNG_WR_STOP; - ODP_DBG("Close %s for pcap tracing\n", event->name); + _ODP_DBG("Close %s for pcap tracing\n", event->name); } else { - ODP_ERR("Unknown inotify event 0x%08x\n", event->mask); + _ODP_ERR("Unknown inotify event 0x%08x\n", event->mask); } } @@ -294,7 +295,7 @@ int _odp_pcapng_start(pktio_entry_t *entry) fifo_sz = get_fifo_max_size(); if (fifo_sz < 0) - ODP_DBG("failed to read max fifo size\n"); + _ODP_DBG("failed to read max fifo size\n"); for (i = 0; i < max_queue; i++) { char pcapng_name[128]; @@ -308,29 +309,28 @@ int _odp_pcapng_start(pktio_entry_t *entry) snprintf(pcapng_path, sizeof(pcapng_path), "%s/%s", PCAPNG_WATCH_DIR, pcapng_name); if (mkfifo(pcapng_path, O_RDWR)) { - ODP_ERR("pcap not available for %s %s\n", - pcapng_path, strerror(errno)); + _ODP_ERR("pcap not available for %s %s\n", pcapng_path, strerror(errno)); continue; } if (chmod(pcapng_path, S_IRUSR | S_IRGRP)) - ODP_ERR("Failed to change file permission for %s %s\n", - pcapng_path, strerror(errno)); + _ODP_ERR("Failed to change file permission for %s %s\n", + pcapng_path, strerror(errno)); fd = open(pcapng_path, O_RDWR | O_NONBLOCK); if (fd == -1) { - ODP_ERR("Fail to open fifo\n"); + _ODP_ERR("Fail to open fifo\n"); entry->pcapng.state[i] = PCAPNG_WR_STOP; if (remove(pcapng_path) == -1) - ODP_ERR("Can't remove fifo %s\n", pcapng_path); + _ODP_ERR("Can't remove fifo %s\n", pcapng_path); continue; } if (fifo_sz > 0) { if (fcntl(fd, F_SETPIPE_SZ, fifo_sz) != fifo_sz) - ODP_DBG("Failed to set max fifo size\n"); + _ODP_DBG("Failed to set max fifo size\n"); else - ODP_DBG("set pcap fifo size %i\n", fifo_sz); + _ODP_DBG("set pcap fifo size %i\n", fifo_sz); } entry->pcapng.fd[i] = fd; @@ -351,7 +351,7 @@ int _odp_pcapng_start(pktio_entry_t *entry) pcapng_gbl->inotify_fd = inotify_init(); if (pcapng_gbl->inotify_fd == -1) { - ODP_ERR("can't init inotify. pcap disabled\n"); + _ODP_ERR("can't init inotify. pcap disabled\n"); goto out_destroy; } @@ -360,8 +360,7 @@ int _odp_pcapng_start(pktio_entry_t *entry) IN_CLOSE | IN_OPEN); if (pcapng_gbl->inotify_watch_fd == -1) { - ODP_ERR("can't register inotify for %s. pcap disabled\n", - strerror(errno)); + _ODP_ERR("can't register inotify for %s. pcap disabled\n", strerror(errno)); goto out_destroy; } @@ -370,7 +369,7 @@ int _odp_pcapng_start(pktio_entry_t *entry) ret = pthread_create(&pcapng_gbl->inotify_thread, &attr, inotify_update, &pcapng_gbl->inotify_fd); if (ret) { - ODP_ERR("Can't start inotify thread (ret=%d). pcapng disabled.\n", ret); + _ODP_ERR("Can't start inotify thread (ret=%d). pcapng disabled.\n", ret); } else { pcapng_gbl->entry[odp_pktio_index(entry->handle)] = entry; pcapng_gbl->num_entries++; @@ -404,8 +403,7 @@ void _odp_pcapng_stop(pktio_entry_t *entry) pcapng_gbl->num_entries == 0) { ret = pthread_cancel(pcapng_gbl->inotify_thread); if (ret) - ODP_ERR("can't cancel inotify thread %s\n", - strerror(errno)); + _ODP_ERR("can't cancel inotify thread %s\n", strerror(errno)); pcapng_gbl->inotify_is_running = 0; } @@ -414,8 +412,7 @@ void _odp_pcapng_stop(pktio_entry_t *entry) ret = inotify_rm_watch(pcapng_gbl->inotify_fd, pcapng_gbl->inotify_watch_fd); if (ret) - ODP_ERR("can't deregister inotify %s\n", - strerror(errno)); + _ODP_ERR("can't deregister inotify %s\n", strerror(errno)); if (pcapng_gbl->inotify_fd != -1) close(pcapng_gbl->inotify_fd); @@ -439,7 +436,7 @@ void _odp_pcapng_stop(pktio_entry_t *entry) PCAPNG_WATCH_DIR, pcapng_name); if (remove(pcapng_path)) - ODP_ERR("can't delete fifo %s\n", pcapng_path); + _ODP_ERR("can't delete fifo %s\n", pcapng_path); } } @@ -464,7 +461,7 @@ int write_pcapng_hdr(pktio_entry_t *entry, int qidx) len = write(fd, &shb, sizeof(shb)); /* fail to write shb/idb means the pcapng is unreadable */ if (len != sizeof(shb)) { - ODP_ERR("Failed to write pcapng section hdr\n"); + _ODP_ERR("Failed to write pcapng section hdr\n"); return -1; } fsync(fd); @@ -476,7 +473,7 @@ int write_pcapng_hdr(pktio_entry_t *entry, int qidx) idb.snaplen = 0x0; /* unlimited */ len = write(fd, &idb, sizeof(idb)); if (len != sizeof(idb)) { - ODP_ERR("Failed to write pcapng interface description\n"); + _ODP_ERR("Failed to write pcapng interface description\n"); return -1; } fsync(fd); diff --git a/platform/linux-generic/odp_pkt_queue.c b/platform/linux-generic/odp_pkt_queue.c index e977efc24..d3dd6639f 100644 --- a/platform/linux-generic/odp_pkt_queue.c +++ b/platform/linux-generic/odp_pkt_queue.c @@ -280,7 +280,7 @@ void _odp_pkt_queue_destroy(_odp_int_queue_pool_t queue_pool, uint32_t queue_num = (uint32_t)pkt_queue; if ((queue_num == 0) || (queue_num > pool->max_queue_num)) { - ODP_ERR("Invalid TM packet queue ID\n"); + _ODP_ERR("Invalid TM packet queue ID\n"); return; } @@ -406,17 +406,17 @@ void _odp_pkt_queue_stats_print(_odp_int_queue_pool_t queue_pool) queue_pool_t *pool; pool = (queue_pool_t *)(uintptr_t)queue_pool; - ODP_PRINT("pkt_queue_stats - queue_pool=0x%" PRIX64 "\n", queue_pool); - ODP_PRINT(" max_queue_num=%" PRIu32 " max_queued_pkts=%" PRIu32 " " - "num_queues=%" PRIu32 "\n", pool->max_queue_num, - pool->max_queued_pkts, pool->num_queues); - ODP_PRINT(" total pkt appends=%" PRIu64 " total pkt removes=%" PRIu64 - " bad removes=%" PRIu64 "\n", - pool->total_pkt_appends, pool->total_pkt_removes, - pool->total_bad_removes); - ODP_PRINT(" free_list size=%u min size=%u peak size=%u\n", - pool->free_list_size, pool->min_free_list_size, - pool->peak_free_list_size); + _ODP_PRINT(" pkt_queue_stats - queue_pool=0x%" PRIX64 "\n", queue_pool); + _ODP_PRINT(" max_queue_num=%" PRIu32 " max_queued_pkts=%" PRIu32 " " + "num_queues=%" PRIu32 "\n", pool->max_queue_num, + pool->max_queued_pkts, pool->num_queues); + _ODP_PRINT(" total pkt appends=%" PRIu64 " total pkt removes=%" PRIu64 + " bad removes=%" PRIu64 "\n", + pool->total_pkt_appends, pool->total_pkt_removes, + pool->total_bad_removes); + _ODP_PRINT(" free_list size=%u min size=%u peak size=%u\n", + pool->free_list_size, pool->min_free_list_size, + pool->peak_free_list_size); } void _odp_queue_pool_destroy(_odp_int_queue_pool_t queue_pool) diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index a92cc615d..f414b0626 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -75,6 +75,7 @@ static __thread pool_local_t local; /* Fill in pool header field offsets for inline functions */ const _odp_pool_inline_offset_t _odp_pool_inline ODP_ALIGNED_CACHE = { .index = offsetof(pool_t, pool_idx), + .seg_len = offsetof(pool_t, seg_len), .uarea_size = offsetof(pool_t, param_uarea_size), .ext_head_offset = offsetof(pool_t, ext_head_offset), .ext_pkt_buf_size = offsetof(pool_t, ext_param.pkt.buf_size) @@ -145,13 +146,13 @@ static inline int cache_available(pool_t *pool, odp_pool_stats_t *stats) if (per_thread) { if (first > last || last >= odp_thread_count_max()) { - ODP_ERR("Bad thread ids: first=%" PRIu16 " last=%" PRIu16 "\n", - first, last); + _ODP_ERR("Bad thread ids: first=%" PRIu16 " last=%" PRIu16 "\n", + first, last); return -1; } if (last - first + 1 > ODP_POOL_MAX_THREAD_STATS) { - ODP_ERR("Too many thread ids: max=%d\n", ODP_POOL_MAX_THREAD_STATS); + _ODP_ERR("Too many thread ids: max=%d\n", ODP_POOL_MAX_THREAD_STATS); return -1; } } @@ -177,81 +178,80 @@ static int read_config_file(pool_global_t *pool_glb) const char *str; int val = 0; - ODP_PRINT("Pool config:\n"); + _ODP_PRINT("Pool config:\n"); str = "pool.local_cache_size"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } if (val > CONFIG_POOL_CACHE_MAX_SIZE || val < 0) { - ODP_ERR("Bad value %s = %i, max %i\n", str, val, - CONFIG_POOL_CACHE_MAX_SIZE); + _ODP_ERR("Bad value %s = %i, max %i\n", str, val, CONFIG_POOL_CACHE_MAX_SIZE); return -1; } local_cache_size = val; pool_glb->config.local_cache_size = local_cache_size; - ODP_PRINT(" %s: %i\n", str, val); + _ODP_PRINT(" %s: %i\n", str, val); str = "pool.burst_size"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } if (val <= 0) { - ODP_ERR("Bad value %s = %i\n", str, val); + _ODP_ERR("Bad value %s = %i\n", str, val); return -1; } burst_size = val; pool_glb->config.burst_size = burst_size; - ODP_PRINT(" %s: %i\n", str, val); + _ODP_PRINT(" %s: %i\n", str, val); /* Check local cache size and burst size relation */ if (local_cache_size % burst_size) { - ODP_ERR("Pool cache size not multiple of burst size\n"); + _ODP_ERR("Pool cache size not multiple of burst size\n"); return -1; } if (local_cache_size && (local_cache_size / burst_size < 2)) { - ODP_ERR("Cache burst size too large compared to cache size\n"); + _ODP_ERR("Cache burst size too large compared to cache size\n"); return -1; } str = "pool.pkt.max_num"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } if (val > CONFIG_POOL_MAX_NUM || val < POOL_MAX_NUM_MIN) { - ODP_ERR("Bad value %s = %i\n", str, val); + _ODP_ERR("Bad value %s = %i\n", str, val); return -1; } pool_glb->config.pkt_max_num = val; - ODP_PRINT(" %s: %i\n", str, val); + _ODP_PRINT(" %s: %i\n", str, val); str = "pool.pkt.max_len"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } if (val <= 0) { - ODP_ERR("Bad value %s = %i\n", str, val); + _ODP_ERR("Bad value %s = %i\n", str, val); return -1; } pool_glb->config.pkt_max_len = val; - ODP_PRINT(" %s: %i\n", str, val); + _ODP_PRINT(" %s: %i\n", str, val); str = "pool.pkt.base_align"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } @@ -260,16 +260,16 @@ static int read_config_file(pool_global_t *pool_glb) align = ODP_CACHE_LINE_SIZE; if (!_ODP_CHECK_IS_POWER2(align)) { - ODP_ERR("Not a power of two: %s = %i\n", str, val); + _ODP_ERR("Not a power of two: %s = %i\n", str, val); return -1; } pool_glb->config.pkt_base_align = align; - ODP_PRINT(" %s: %u\n", str, align); + _ODP_PRINT(" %s: %u\n", str, align); str = "pool.buf.min_align"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } @@ -278,14 +278,14 @@ static int read_config_file(pool_global_t *pool_glb) align = ODP_CACHE_LINE_SIZE; if (!_ODP_CHECK_IS_POWER2(align)) { - ODP_ERR("Not a power of two: %s = %i\n", str, val); + _ODP_ERR("Not a power of two: %s = %i\n", str, val); return -1; } pool_glb->config.buf_min_align = align; - ODP_PRINT(" %s: %u\n", str, align); + _ODP_PRINT(" %s: %u\n", str, align); - ODP_PRINT("\n"); + _ODP_PRINT("\n"); return 0; } @@ -321,15 +321,15 @@ int _odp_pool_init_global(void) pool->pool_idx = i; } - ODP_DBG("\nPool init global\n"); - ODP_DBG(" event_hdr_t size %zu\n", sizeof(_odp_event_hdr_t)); - ODP_DBG(" buffer_hdr_t size %zu\n", sizeof(odp_buffer_hdr_t)); - ODP_DBG(" packet_hdr_t size %zu\n", sizeof(odp_packet_hdr_t)); - ODP_DBG(" timeout_hdr_t size %zu\n", sizeof(odp_timeout_hdr_t)); - ODP_DBG(" event_vector_hdr_t size %zu\n", sizeof(odp_event_vector_hdr_t)); - ODP_DBG(" packet_hdr_t::seg_data offset %zu\n", offsetof(odp_packet_hdr_t, seg_data)); - ODP_DBG(" packet_hdr_t::timestamp offset %zu\n", offsetof(odp_packet_hdr_t, timestamp)); - ODP_DBG("\n"); + _ODP_DBG("\nPool init global\n"); + _ODP_DBG(" event_hdr_t size %zu\n", sizeof(_odp_event_hdr_t)); + _ODP_DBG(" buffer_hdr_t size %zu\n", sizeof(odp_buffer_hdr_t)); + _ODP_DBG(" packet_hdr_t size %zu\n", sizeof(odp_packet_hdr_t)); + _ODP_DBG(" timeout_hdr_t size %zu\n", sizeof(odp_timeout_hdr_t)); + _ODP_DBG(" event_vector_hdr_t size %zu\n", sizeof(odp_event_vector_hdr_t)); + _ODP_DBG(" packet_hdr_t::seg_data offset %zu\n", offsetof(odp_packet_hdr_t, seg_data)); + _ODP_DBG(" packet_hdr_t::timestamp offset %zu\n", offsetof(odp_packet_hdr_t, timestamp)); + _ODP_DBG("\n"); return 0; } @@ -348,7 +348,7 @@ int _odp_pool_term_global(void) LOCK(&pool->lock); if (pool->reserved) { - ODP_ERR("Not destroyed pool: %s\n", pool->name); + _ODP_ERR("Not destroyed pool: %s\n", pool->name); rc = -1; } UNLOCK(&pool->lock); @@ -356,7 +356,7 @@ int _odp_pool_term_global(void) ret = odp_shm_free(_odp_pool_glb->shm); if (ret < 0) { - ODP_ERR("SHM free failed\n"); + _ODP_ERR("SHM free failed\n"); rc = -1; } @@ -422,7 +422,7 @@ static pool_t *reserve_pool(uint32_t shmflags, uint8_t pool_ext, uint32_t num) shm = odp_shm_reserve(ring_name, mem_size, ODP_CACHE_LINE_SIZE, shmflags); if (odp_unlikely(shm == ODP_SHM_INVALID)) { - ODP_ERR("Unable to alloc pool ring %d\n", i); + _ODP_ERR("Unable to alloc pool ring %d\n", i); LOCK(&pool->lock); pool->reserved = 0; UNLOCK(&pool->lock); @@ -529,7 +529,7 @@ static void init_buffers(pool_t *pool) int skipped_blocks = 0; if (odp_shm_info(pool->shm, &shm_info)) - ODP_ABORT("Shm info failed\n"); + _ODP_ABORT("Shm info failed\n"); page_size = shm_info.page_size; ring = &pool->ring->hdr; @@ -600,7 +600,7 @@ static bool shm_is_from_huge_pages(odp_shm_t shm) return 0; if (odp_shm_info(shm, &info)) { - ODP_ERR("Failed to fetch shm info\n"); + _ODP_ERR("Failed to fetch shm info\n"); return 0; } @@ -679,8 +679,8 @@ static void set_mem_src_ops(pool_t *pool) if (_odp_pool_mem_src_ops[i]->is_active()) { is_active_found = true; pool->mem_src_ops = _odp_pool_mem_src_ops[i]; - ODP_DBG("Packet pool as a memory source for: %s\n", - pool->mem_src_ops->name); + _ODP_DBG("Packet pool as a memory source for: %s\n", + pool->mem_src_ops->name); } } else if (_odp_pool_mem_src_ops[i]->is_active()) { _odp_pool_mem_src_ops[i]->force_disable(); @@ -719,7 +719,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, if (align_req && (!_ODP_CHECK_IS_POWER2(align_req) || align_req > _odp_pool_glb->config.pkt_base_align)) { - ODP_ERR("Bad align requirement\n"); + _ODP_ERR("Bad align requirement\n"); return ODP_POOL_INVALID; } @@ -735,7 +735,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, /* Validate requested buffer alignment */ if (align > ODP_CONFIG_BUFFER_ALIGN_MAX || align != _ODP_ROUNDDOWN_POWER2(align, align)) { - ODP_ERR("Bad align requirement\n"); + _ODP_ERR("Bad align requirement\n"); return ODP_POOL_INVALID; } @@ -756,7 +756,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, case ODP_POOL_PACKET: if (params->pkt.headroom > CONFIG_PACKET_HEADROOM) { - ODP_ERR("Packet headroom size not supported\n"); + _ODP_ERR("Packet headroom size not supported\n"); return ODP_POOL_INVALID; } @@ -779,7 +779,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, if ((max_len + seg_len - 1) / seg_len > PKT_MAX_SEGS) seg_len = (max_len + PKT_MAX_SEGS - 1) / PKT_MAX_SEGS; if (seg_len > CONFIG_PACKET_MAX_SEG_LEN) { - ODP_ERR("Pool unable to store 'max_len' packet\n"); + _ODP_ERR("Pool unable to store 'max_len' packet\n"); return ODP_POOL_INVALID; } @@ -789,8 +789,8 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, /* Make sure 'params->pkt.max_num' limitation holds */ if (params->pkt.max_num && num > params->pkt.max_num) { - ODP_ERR("Pool 'max_num' parameter too small (%u/%u)\n", - params->pkt.max_num, num); + _ODP_ERR("Pool 'max_num' parameter too small (%u/%u)\n", + params->pkt.max_num, num); return ODP_POOL_INVALID; } @@ -814,14 +814,14 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, break; default: - ODP_ERR("Bad pool type\n"); + _ODP_ERR("Bad pool type\n"); return ODP_POOL_INVALID; } pool = reserve_pool(shmflags, 0, num); if (pool == NULL) { - ODP_ERR("No more free pools\n"); + _ODP_ERR("No more free pools\n"); return ODP_POOL_INVALID; } @@ -848,7 +848,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, &pool->block_offset, &shmflags); if (!adj_size) { - ODP_ERR("Calculating adjusted block size failed\n"); + _ODP_ERR("Calculating adjusted block size failed\n"); return ODP_POOL_INVALID; } } @@ -908,7 +908,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, pool->shm = shm; if (shm == ODP_SHM_INVALID) { - ODP_ERR("SHM reserve failed\n"); + _ODP_ERR("SHM reserve failed\n"); goto error; } @@ -918,7 +918,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, pool->max_addr = pool->base_addr + pool->shm_size - 1; if (reserve_uarea(pool, uarea_size, num, shmflags)) { - ODP_ERR("User area SHM reserve failed\n"); + _ODP_ERR("User area SHM reserve failed\n"); goto error; } @@ -927,7 +927,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, if (type == ODP_POOL_PACKET && pool->mem_src_ops && pool->mem_src_ops->bind && pool->mem_src_ops->bind(pool->mem_src_data, pool)) { - ODP_ERR("Binding pool as memory source failed\n"); + _ODP_ERR("Binding pool as memory source failed\n"); goto error; } @@ -981,27 +981,27 @@ static int check_params(const odp_pool_param_t *params) cache_size = params->buf.cache_size; if (params->buf.num > capa.buf.max_num) { - ODP_ERR("buf.num too large %u\n", params->buf.num); + _ODP_ERR("buf.num too large %u\n", params->buf.num); return -1; } if (params->buf.size > capa.buf.max_size) { - ODP_ERR("buf.size too large %u\n", params->buf.size); + _ODP_ERR("buf.size too large %u\n", params->buf.size); return -1; } if (params->buf.align > capa.buf.max_align) { - ODP_ERR("buf.align too large %u\n", params->buf.align); + _ODP_ERR("buf.align too large %u\n", params->buf.align); return -1; } if (params->buf.uarea_size > capa.buf.max_uarea_size) { - ODP_ERR("buf.uarea_size too large %u\n", params->buf.uarea_size); + _ODP_ERR("buf.uarea_size too large %u\n", params->buf.uarea_size); return -1; } if (params->stats.all & ~capa.buf.stats.all) { - ODP_ERR("Unsupported pool statistics counter\n"); + _ODP_ERR("Unsupported pool statistics counter\n"); return -1; } @@ -1012,47 +1012,42 @@ static int check_params(const odp_pool_param_t *params) cache_size = params->pkt.cache_size; if (params->pkt.num > capa.pkt.max_num) { - ODP_ERR("pkt.num too large %u\n", params->pkt.num); + _ODP_ERR("pkt.num too large %u\n", params->pkt.num); return -1; } if (params->pkt.max_num > capa.pkt.max_num) { - ODP_ERR("pkt.max_num too large %u\n", - params->pkt.max_num); + _ODP_ERR("pkt.max_num too large %u\n", params->pkt.max_num); return -1; } if (params->pkt.len > capa.pkt.max_len) { - ODP_ERR("pkt.len too large %u\n", params->pkt.len); + _ODP_ERR("pkt.len too large %u\n", params->pkt.len); return -1; } if (params->pkt.max_len > capa.pkt.max_len) { - ODP_ERR("pkt.max_len too large %u\n", - params->pkt.max_len); + _ODP_ERR("pkt.max_len too large %u\n", params->pkt.max_len); return -1; } if (params->pkt.seg_len > capa.pkt.max_seg_len) { - ODP_ERR("pkt.seg_len too large %u\n", - params->pkt.seg_len); + _ODP_ERR("pkt.seg_len too large %u\n", params->pkt.seg_len); return -1; } if (params->pkt.uarea_size > capa.pkt.max_uarea_size) { - ODP_ERR("pkt.uarea_size too large %u\n", - params->pkt.uarea_size); + _ODP_ERR("pkt.uarea_size too large %u\n", params->pkt.uarea_size); return -1; } if (params->pkt.headroom > capa.pkt.max_headroom) { - ODP_ERR("pkt.headroom too large %u\n", - params->pkt.headroom); + _ODP_ERR("pkt.headroom too large %u\n", params->pkt.headroom); return -1; } if (params->stats.all & ~capa.pkt.stats.all) { - ODP_ERR("Unsupported pool statistics counter\n"); + _ODP_ERR("Unsupported pool statistics counter\n"); return -1; } @@ -1063,17 +1058,17 @@ static int check_params(const odp_pool_param_t *params) cache_size = params->tmo.cache_size; if (params->tmo.num > capa.tmo.max_num) { - ODP_ERR("tmo.num too large %u\n", params->tmo.num); + _ODP_ERR("tmo.num too large %u\n", params->tmo.num); return -1; } if (params->tmo.uarea_size > capa.tmo.max_uarea_size) { - ODP_ERR("tmo.uarea_size too large %u\n", params->tmo.uarea_size); + _ODP_ERR("tmo.uarea_size too large %u\n", params->tmo.uarea_size); return -1; } if (params->stats.all & ~capa.tmo.stats.all) { - ODP_ERR("Unsupported pool statistics counter\n"); + _ODP_ERR("Unsupported pool statistics counter\n"); return -1; } @@ -1084,49 +1079,49 @@ static int check_params(const odp_pool_param_t *params) cache_size = params->vector.cache_size; if (params->vector.num == 0) { - ODP_ERR("vector.num zero\n"); + _ODP_ERR("vector.num zero\n"); return -1; } if (params->vector.num > capa.vector.max_num) { - ODP_ERR("vector.num too large %u\n", params->vector.num); + _ODP_ERR("vector.num too large %u\n", params->vector.num); return -1; } if (params->vector.max_size == 0) { - ODP_ERR("vector.max_size zero\n"); + _ODP_ERR("vector.max_size zero\n"); return -1; } if (params->vector.max_size > capa.vector.max_size) { - ODP_ERR("vector.max_size too large %u\n", params->vector.max_size); + _ODP_ERR("vector.max_size too large %u\n", params->vector.max_size); return -1; } if (params->vector.uarea_size > capa.vector.max_uarea_size) { - ODP_ERR("vector.uarea_size too large %u\n", params->vector.uarea_size); + _ODP_ERR("vector.uarea_size too large %u\n", params->vector.uarea_size); return -1; } if (params->stats.all & ~capa.vector.stats.all) { - ODP_ERR("Unsupported pool statistics counter\n"); + _ODP_ERR("Unsupported pool statistics counter\n"); return -1; } break; default: - ODP_ERR("bad pool type %i\n", params->type); + _ODP_ERR("bad pool type %i\n", params->type); return -1; } if (cache_size > CONFIG_POOL_CACHE_MAX_SIZE) { - ODP_ERR("Too large cache size %u\n", cache_size); + _ODP_ERR("Too large cache size %u\n", cache_size); return -1; } if (num <= (num_threads * cache_size)) - ODP_DBG("Entire pool fits into thread local caches. Pool " + _ODP_DBG("Entire pool fits into thread local caches. Pool " "starvation may occur if the pool is used by multiple " "threads.\n"); @@ -1153,7 +1148,7 @@ int odp_pool_destroy(odp_pool_t pool_hdl) if (pool->reserved == 0) { UNLOCK(&pool->lock); - ODP_ERR("Pool not created\n"); + _ODP_ERR("Pool not created\n"); return -1; } @@ -1382,11 +1377,11 @@ odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl) pool_t *pool; int ret; - ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); + _ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); pool = _odp_pool_entry(pool_hdl); - ODP_ASSERT(pool->type == ODP_POOL_BUFFER); + _ODP_ASSERT(pool->type == ODP_POOL_BUFFER); ret = _odp_event_alloc_multi(pool, (_odp_event_hdr_t **)&buf, 1); @@ -1413,11 +1408,11 @@ int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num) { pool_t *pool; - ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); + _ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); pool = _odp_pool_entry(pool_hdl); - ODP_ASSERT(pool->type == ODP_POOL_BUFFER); + _ODP_ASSERT(pool->type == ODP_POOL_BUFFER); return _odp_event_alloc_multi(pool, (_odp_event_hdr_t **)buf, num); } @@ -1505,39 +1500,37 @@ void odp_pool_print(odp_pool_t pool_hdl) pool = _odp_pool_entry(pool_hdl); - ODP_PRINT("\nPool info\n"); - ODP_PRINT("---------\n"); - ODP_PRINT(" pool %" PRIu64 "\n", - odp_pool_to_u64(_odp_pool_handle(pool))); - ODP_PRINT(" name %s\n", pool->name); - ODP_PRINT(" pool type %s\n", - pool->type == ODP_POOL_BUFFER ? "buffer" : - (pool->type == ODP_POOL_PACKET ? "packet" : - (pool->type == ODP_POOL_TIMEOUT ? "timeout" : - (pool->type == ODP_POOL_VECTOR ? "vector" : - "unknown")))); - ODP_PRINT(" pool shm %" PRIu64 "\n", - odp_shm_to_u64(pool->shm)); - ODP_PRINT(" user area shm %" PRIu64 "\n", - odp_shm_to_u64(pool->uarea_shm)); - ODP_PRINT(" num %u\n", pool->num); - ODP_PRINT(" align %u\n", pool->align); - ODP_PRINT(" headroom %u\n", pool->headroom); - ODP_PRINT(" seg len %u\n", pool->seg_len); - ODP_PRINT(" max data len %u\n", pool->max_len); - ODP_PRINT(" tailroom %u\n", pool->tailroom); - ODP_PRINT(" block size %u\n", pool->block_size); - ODP_PRINT(" uarea size %u\n", pool->uarea_size); - ODP_PRINT(" shm size %" PRIu64 "\n", pool->shm_size); - ODP_PRINT(" base addr %p\n", (void *)pool->base_addr); - ODP_PRINT(" max addr %p\n", (void *)pool->max_addr); - ODP_PRINT(" uarea shm size %" PRIu64 "\n", pool->uarea_shm_size); - ODP_PRINT(" uarea base addr %p\n", (void *)pool->uarea_base_addr); - ODP_PRINT(" cache size %u\n", pool->cache_size); - ODP_PRINT(" burst size %u\n", pool->burst_size); - ODP_PRINT(" mem src %s\n", - pool->mem_src_ops ? pool->mem_src_ops->name : "(none)"); - ODP_PRINT("\n"); + _ODP_PRINT("\nPool info\n"); + _ODP_PRINT("---------\n"); + _ODP_PRINT(" pool %" PRIu64 "\n", + odp_pool_to_u64(_odp_pool_handle(pool))); + _ODP_PRINT(" name %s\n", pool->name); + _ODP_PRINT(" pool type %s\n", + pool->type == ODP_POOL_BUFFER ? "buffer" : + (pool->type == ODP_POOL_PACKET ? "packet" : + (pool->type == ODP_POOL_TIMEOUT ? "timeout" : + (pool->type == ODP_POOL_VECTOR ? "vector" : + "unknown")))); + _ODP_PRINT(" pool shm %" PRIu64 "\n", odp_shm_to_u64(pool->shm)); + _ODP_PRINT(" user area shm %" PRIu64 "\n", odp_shm_to_u64(pool->uarea_shm)); + _ODP_PRINT(" num %u\n", pool->num); + _ODP_PRINT(" align %u\n", pool->align); + _ODP_PRINT(" headroom %u\n", pool->headroom); + _ODP_PRINT(" seg len %u\n", pool->seg_len); + _ODP_PRINT(" max data len %u\n", pool->max_len); + _ODP_PRINT(" tailroom %u\n", pool->tailroom); + _ODP_PRINT(" block size %u\n", pool->block_size); + _ODP_PRINT(" uarea size %u\n", pool->uarea_size); + _ODP_PRINT(" shm size %" PRIu64 "\n", pool->shm_size); + _ODP_PRINT(" base addr %p\n", (void *)pool->base_addr); + _ODP_PRINT(" max addr %p\n", (void *)pool->max_addr); + _ODP_PRINT(" uarea shm size %" PRIu64 "\n", pool->uarea_shm_size); + _ODP_PRINT(" uarea base addr %p\n", (void *)pool->uarea_base_addr); + _ODP_PRINT(" cache size %u\n", pool->cache_size); + _ODP_PRINT(" burst size %u\n", pool->burst_size); + _ODP_PRINT(" mem src %s\n", + pool->mem_src_ops ? pool->mem_src_ops->name : "(none)"); + _ODP_PRINT("\n"); } void odp_pool_print_all(void) @@ -1550,9 +1543,9 @@ void odp_pool_print_all(void) const char *name; char type_c; - ODP_PRINT("\nList of all pools\n"); - ODP_PRINT("-----------------\n"); - ODP_PRINT(" idx %-*s type free tot cache buf_len ext\n", col_width, "name"); + _ODP_PRINT("\nList of all pools\n"); + _ODP_PRINT("-----------------\n"); + _ODP_PRINT(" idx %-*s type free tot cache buf_len ext\n", col_width, "name"); for (i = 0; i < ODP_CONFIG_POOLS; i++) { pool_t *pool = _odp_pool_entry_from_idx(i); @@ -1583,11 +1576,11 @@ void odp_pool_print_all(void) (type == ODP_POOL_TIMEOUT) ? 'T' : (type == ODP_POOL_VECTOR) ? 'V' : '-'; - ODP_PRINT("%4u %-*s %c %6" PRIu64 " %6" PRIu32 " %6" PRIu32 " %8" PRIu32 " " + _ODP_PRINT("%4u %-*s %c %6" PRIu64 " %6" PRIu32 " %6" PRIu32 " %8" PRIu32 " " "%" PRIu8 "\n", index, col_width, name, type_c, available, tot, cache_size, buf_len, ext); } - ODP_PRINT("\n"); + _ODP_PRINT("\n"); } void odp_pool_param_init(odp_pool_param_t *params) @@ -1618,11 +1611,11 @@ int odp_pool_stats(odp_pool_t pool_hdl, odp_pool_stats_t *stats) uint16_t first, last; if (odp_unlikely(pool_hdl == ODP_POOL_INVALID)) { - ODP_ERR("Invalid pool handle\n"); + _ODP_ERR("Invalid pool handle\n"); return -1; } if (odp_unlikely(stats == NULL)) { - ODP_ERR("Output buffer NULL\n"); + _ODP_ERR("Output buffer NULL\n"); return -1; } @@ -1671,7 +1664,7 @@ int odp_pool_stats_reset(odp_pool_t pool_hdl) pool_t *pool; if (odp_unlikely(pool_hdl == ODP_POOL_INVALID)) { - ODP_ERR("Invalid pool handle\n"); + _ODP_ERR("Invalid pool handle\n"); return -1; } @@ -1793,47 +1786,47 @@ static int check_pool_ext_param(const odp_pool_ext_param_t *param) uint32_t head_offset = sizeof(odp_packet_hdr_t) + param->pkt.app_header_size; if (param->type != ODP_POOL_PACKET) { - ODP_ERR("Pool type not supported\n"); + _ODP_ERR("Pool type not supported\n"); return -1; } if (odp_pool_ext_capability(param->type, &capa)) { - ODP_ERR("Capa failed\n"); + _ODP_ERR("Capa failed\n"); return -1; } if (param->cache_size > capa.max_cache_size) { - ODP_ERR("Too large cache size %u\n", param->cache_size); + _ODP_ERR("Too large cache size %u\n", param->cache_size); return -1; } if (param->stats.all != capa.stats.all) { - ODP_ERR("Pool statistics not supported\n"); + _ODP_ERR("Pool statistics not supported\n"); return -1; } if (param->pkt.num_buf > capa.pkt.max_num_buf) { - ODP_ERR("Too many packet buffers\n"); + _ODP_ERR("Too many packet buffers\n"); return -1; } if (param->pkt.buf_size > capa.pkt.max_buf_size) { - ODP_ERR("Too large packet buffer size %u\n", param->pkt.buf_size); + _ODP_ERR("Too large packet buffer size %u\n", param->pkt.buf_size); return -1; } if (param->pkt.uarea_size > capa.pkt.max_uarea_size) { - ODP_ERR("Too large user area size %u\n", param->pkt.uarea_size); + _ODP_ERR("Too large user area size %u\n", param->pkt.uarea_size); return -1; } if (param->pkt.headroom > capa.pkt.max_headroom) { - ODP_ERR("Too large headroom size\n"); + _ODP_ERR("Too large headroom size\n"); return -1; } if (head_offset % capa.pkt.min_head_align) { - ODP_ERR("Head pointer not %u byte aligned\n", capa.pkt.min_head_align); + _ODP_ERR("Head pointer not %u byte aligned\n", capa.pkt.min_head_align); return -1; } @@ -1851,7 +1844,7 @@ odp_pool_t odp_pool_ext_create(const char *name, const odp_pool_ext_param_t *par uint32_t shm_flags = 0; if (check_pool_ext_param(param)) { - ODP_ERR("Bad pool ext param\n"); + _ODP_ERR("Bad pool ext param\n"); return ODP_POOL_INVALID; } @@ -1861,7 +1854,7 @@ odp_pool_t odp_pool_ext_create(const char *name, const odp_pool_ext_param_t *par pool = reserve_pool(shm_flags, 1, num_buf); if (pool == NULL) { - ODP_ERR("No more free pools\n"); + _ODP_ERR("No more free pools\n"); return ODP_POOL_INVALID; } @@ -1870,7 +1863,7 @@ odp_pool_t odp_pool_ext_create(const char *name, const odp_pool_ext_param_t *par set_pool_cache_size(pool, param->cache_size); if (reserve_uarea(pool, param->pkt.uarea_size, num_buf, shm_flags)) { - ODP_ERR("User area SHM reserve failed\n"); + _ODP_ERR("User area SHM reserve failed\n"); goto error; } @@ -1919,14 +1912,14 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, u void *uarea = NULL; if (pool_hdl == ODP_POOL_INVALID) { - ODP_ERR("Bad pool handle\n"); + _ODP_ERR("Bad pool handle\n"); return -1; } pool = _odp_pool_entry(pool_hdl); if (pool->type != ODP_POOL_PACKET || pool->pool_ext == 0) { - ODP_ERR("Bad pool type\n"); + _ODP_ERR("Bad pool type\n"); return -1; } @@ -1934,24 +1927,24 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, u max_addr = pool->max_addr; if (buf_size != pool->ext_param.pkt.buf_size) { - ODP_ERR("Bad buffer size\n"); + _ODP_ERR("Bad buffer size\n"); return -1; } num_populated = pool->num_populated; if (num_populated + num > pool->num) { - ODP_ERR("Trying to over populate the pool\n"); + _ODP_ERR("Trying to over populate the pool\n"); return -1; } if ((num_populated + num == pool->num) && !(flags & ODP_POOL_POPULATE_DONE)) { - ODP_ERR("Missing ODP_POOL_POPULATE_DONE flag\n"); + _ODP_ERR("Missing ODP_POOL_POPULATE_DONE flag\n"); return -1; } if ((num_populated + num < pool->num) && flags) { - ODP_ERR("Unexpected flags: 0x%x\n", flags); + _ODP_ERR("Unexpected flags: 0x%x\n", flags); return -1; } @@ -1970,12 +1963,12 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, u max_addr = (uint8_t *)event_hdr; if ((uintptr_t)event_hdr & (ODP_CACHE_LINE_SIZE - 1)) { - ODP_ERR("Bad packet buffer align: buf[%u]\n", i); + _ODP_ERR("Bad packet buffer align: buf[%u]\n", i); return -1; } if (((uintptr_t)event_hdr + head_offset) & (MIN_HEAD_ALIGN - 1)) { - ODP_ERR("Bad head pointer align: buf[%u]\n", i); + _ODP_ERR("Bad head pointer align: buf[%u]\n", i); return -1; } diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index f485b7819..360f907bf 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -67,11 +67,11 @@ static int read_config_file(queue_global_t *_odp_queue_glb) uint32_t val_u32; int val = 0; - ODP_PRINT("Queue config:\n"); + _ODP_PRINT("Queue config:\n"); str = "queue_basic.max_queue_size"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } @@ -79,16 +79,16 @@ static int read_config_file(queue_global_t *_odp_queue_glb) if (val_u32 > MAX_QUEUE_SIZE || val_u32 < MIN_QUEUE_SIZE || !_ODP_CHECK_IS_POWER2(val_u32)) { - ODP_ERR("Bad value %s = %u\n", str, val_u32); + _ODP_ERR("Bad value %s = %u\n", str, val_u32); return -1; } _odp_queue_glb->config.max_queue_size = val_u32; - ODP_PRINT(" %s: %u\n", str, val_u32); + _ODP_PRINT(" %s: %u\n", str, val_u32); str = "queue_basic.default_queue_size"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } @@ -97,12 +97,12 @@ static int read_config_file(queue_global_t *_odp_queue_glb) if (val_u32 > _odp_queue_glb->config.max_queue_size || val_u32 < MIN_QUEUE_SIZE || !_ODP_CHECK_IS_POWER2(val_u32)) { - ODP_ERR("Bad value %s = %u\n", str, val_u32); + _ODP_ERR("Bad value %s = %u\n", str, val_u32); return -1; } _odp_queue_glb->config.default_queue_size = val_u32; - ODP_PRINT(" %s: %u\n\n", str, val_u32); + _ODP_PRINT(" %s: %u\n\n", str, val_u32); return 0; } @@ -116,7 +116,7 @@ static int queue_init_global(void) odp_queue_capability_t capa; uint64_t mem_size; - ODP_DBG("Starts...\n"); + _ODP_DBG("Starts...\n"); /* Fill in queue entry field offsets for inline functions */ memset(&_odp_queue_inline_offset, 0, @@ -171,12 +171,12 @@ static int queue_init_global(void) queue_capa(&capa, 0); - ODP_DBG("... done.\n"); - ODP_DBG(" queue_entry_t size %zu\n", sizeof(queue_entry_t)); - ODP_DBG(" max num queues %u\n", capa.max_queues); - ODP_DBG(" max queue size %u\n", capa.plain.max_size); - ODP_DBG(" max num lockfree %u\n", capa.plain.lockfree.max_num); - ODP_DBG(" max lockfree size %u\n\n", capa.plain.lockfree.max_size); + _ODP_DBG("... done.\n"); + _ODP_DBG(" queue_entry_t size %zu\n", sizeof(queue_entry_t)); + _ODP_DBG(" max num queues %u\n", capa.max_queues); + _ODP_DBG(" max queue size %u\n", capa.plain.max_size); + _ODP_DBG(" max num lockfree %u\n", capa.plain.lockfree.max_num); + _ODP_DBG(" max lockfree size %u\n\n", capa.plain.lockfree.max_size); return 0; } @@ -201,7 +201,7 @@ static int queue_term_global(void) queue = qentry_from_index(i); LOCK(queue); if (queue->status != QUEUE_STATUS_FREE) { - ODP_ERR("Not destroyed queue: %s\n", queue->name); + _ODP_ERR("Not destroyed queue: %s\n", queue->name); ret = -1; } UNLOCK(queue); @@ -210,12 +210,12 @@ static int queue_term_global(void) _odp_queue_lf_term_global(); if (odp_shm_free(_odp_queue_glb->queue_ring_shm)) { - ODP_ERR("shm free failed"); + _ODP_ERR("shm free failed"); ret = -1; } if (odp_shm_free(_odp_queue_glb->queue_gbl_shm)) { - ODP_ERR("shm free failed"); + _ODP_ERR("shm free failed"); ret = -1; } @@ -276,7 +276,7 @@ static odp_queue_t queue_create(const char *name, if (type == ODP_QUEUE_TYPE_SCHED) { if (param->sched.prio < odp_schedule_min_prio() || param->sched.prio > odp_schedule_max_prio()) { - ODP_ERR("Bad queue priority: %i\n", param->sched.prio); + _ODP_ERR("Bad queue priority: %i\n", param->sched.prio); return ODP_QUEUE_INVALID; } } @@ -359,7 +359,7 @@ static odp_queue_t queue_create(const char *name, if (_odp_sched_fn->create_queue(queue->index, &queue->param.sched)) { queue->status = QUEUE_STATUS_FREE; - ODP_ERR("schedule queue init failed\n"); + _ODP_ERR("schedule queue init failed\n"); return ODP_QUEUE_INVALID; } } @@ -391,12 +391,12 @@ static int queue_destroy(odp_queue_t handle) LOCK(queue); if (queue->status == QUEUE_STATUS_FREE) { UNLOCK(queue); - ODP_ERR("queue \"%s\" already free\n", queue->name); + _ODP_ERR("queue \"%s\" already free\n", queue->name); return -1; } if (queue->status == QUEUE_STATUS_DESTROYED) { UNLOCK(queue); - ODP_ERR("queue \"%s\" already destroyed\n", queue->name); + _ODP_ERR("queue \"%s\" already destroyed\n", queue->name); return -1; } @@ -409,7 +409,7 @@ static int queue_destroy(odp_queue_t handle) if (!empty) { UNLOCK(queue); - ODP_ERR("queue \"%s\" not empty\n", queue->name); + _ODP_ERR("queue \"%s\" not empty\n", queue->name); return -1; } @@ -426,7 +426,7 @@ static int queue_destroy(odp_queue_t handle) queue->status = QUEUE_STATUS_DESTROYED; break; default: - ODP_ABORT("Unexpected queue status\n"); + _ODP_ABORT("Unexpected queue status\n"); } if (queue->queue_lf) @@ -574,8 +574,7 @@ static int error_enqueue(odp_queue_t handle, _odp_event_hdr_t *event_hdr) { (void)event_hdr; - ODP_ERR("Enqueue not supported (0x%" PRIx64 ")\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Enqueue not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); return -1; } @@ -586,16 +585,14 @@ static int error_enqueue_multi(odp_queue_t handle, (void)event_hdr; (void)num; - ODP_ERR("Enqueue multi not supported (0x%" PRIx64 ")\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Enqueue multi not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); return -1; } static _odp_event_hdr_t *error_dequeue(odp_queue_t handle) { - ODP_ERR("Dequeue not supported (0x%" PRIx64 ")\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Dequeue not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); return NULL; } @@ -606,8 +603,7 @@ static int error_dequeue_multi(odp_queue_t handle, (void)event_hdr; (void)num; - ODP_ERR("Dequeue multi not supported (0x%" PRIx64 ")\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Dequeue multi not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); return -1; } @@ -632,15 +628,14 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info) int status; if (odp_unlikely(info == NULL)) { - ODP_ERR("Unable to store info, NULL ptr given\n"); + _ODP_ERR("Unable to store info, NULL ptr given\n"); return -1; } queue_id = queue_to_index(handle); if (odp_unlikely(queue_id >= CONFIG_MAX_QUEUES)) { - ODP_ERR("Invalid queue handle: 0x%" PRIx64 "\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Invalid queue handle: 0x%" PRIx64 "\n", odp_queue_to_u64(handle)); return -1; } @@ -652,7 +647,7 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info) if (odp_unlikely(status == QUEUE_STATUS_FREE || status == QUEUE_STATUS_DESTROYED)) { UNLOCK(queue); - ODP_ERR("Invalid queue status:%d\n", status); + _ODP_ERR("Invalid queue status:%d\n", status); return -1; } @@ -675,8 +670,7 @@ static void queue_print(odp_queue_t handle) queue_id = queue_to_index(handle); if (odp_unlikely(queue_id >= CONFIG_MAX_QUEUES)) { - ODP_ERR("Invalid queue handle: 0x%" PRIx64 "\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Invalid queue handle: 0x%" PRIx64 "\n", odp_queue_to_u64(handle)); return; } @@ -688,79 +682,78 @@ static void queue_print(odp_queue_t handle) if (odp_unlikely(status == QUEUE_STATUS_FREE || status == QUEUE_STATUS_DESTROYED)) { UNLOCK(queue); - ODP_ERR("Invalid queue status:%d\n", status); + _ODP_ERR("Invalid queue status:%d\n", status); return; } - ODP_PRINT("\nQueue info\n"); - ODP_PRINT("----------\n"); - ODP_PRINT(" handle %p\n", (void *)queue->handle); - ODP_PRINT(" index %" PRIu32 "\n", queue_id); - ODP_PRINT(" name %s\n", queue->name); - ODP_PRINT(" enq mode %s\n", - queue->param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : - (queue->param.enq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : - (queue->param.enq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : - "unknown"))); - ODP_PRINT(" deq mode %s\n", - queue->param.deq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : - (queue->param.deq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : - (queue->param.deq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : - "unknown"))); - ODP_PRINT(" non-blocking %s\n", - queue->param.nonblocking == ODP_BLOCKING ? "ODP_BLOCKING" : - (queue->param.nonblocking == ODP_NONBLOCKING_LF ? "ODP_NONBLOCKING_LF" : - (queue->param.nonblocking == ODP_NONBLOCKING_WF ? "ODP_NONBLOCKING_WF" : - "unknown"))); - ODP_PRINT(" type %s\n", - queue->type == ODP_QUEUE_TYPE_PLAIN ? "ODP_QUEUE_TYPE_PLAIN" : - (queue->type == ODP_QUEUE_TYPE_SCHED ? "ODP_QUEUE_TYPE_SCHED" : "unknown")); + _ODP_PRINT("\nQueue info\n"); + _ODP_PRINT("----------\n"); + _ODP_PRINT(" handle %p\n", (void *)queue->handle); + _ODP_PRINT(" index %" PRIu32 "\n", queue_id); + _ODP_PRINT(" name %s\n", queue->name); + _ODP_PRINT(" enq mode %s\n", + queue->param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : + (queue->param.enq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : + (queue->param.enq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : + "unknown"))); + _ODP_PRINT(" deq mode %s\n", + queue->param.deq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : + (queue->param.deq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : + (queue->param.deq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : + "unknown"))); + _ODP_PRINT(" non-blocking %s\n", + queue->param.nonblocking == ODP_BLOCKING ? "ODP_BLOCKING" : + (queue->param.nonblocking == ODP_NONBLOCKING_LF ? "ODP_NONBLOCKING_LF" : + (queue->param.nonblocking == ODP_NONBLOCKING_WF ? "ODP_NONBLOCKING_WF" : + "unknown"))); + _ODP_PRINT(" type %s\n", + queue->type == ODP_QUEUE_TYPE_PLAIN ? "ODP_QUEUE_TYPE_PLAIN" : + (queue->type == ODP_QUEUE_TYPE_SCHED ? "ODP_QUEUE_TYPE_SCHED" : "unknown")); if (queue->type == ODP_QUEUE_TYPE_SCHED) { - ODP_PRINT(" sync %s\n", - queue->param.sched.sync == ODP_SCHED_SYNC_PARALLEL ? - "ODP_SCHED_SYNC_PARALLEL" : - (queue->param.sched.sync == ODP_SCHED_SYNC_ATOMIC ? - "ODP_SCHED_SYNC_ATOMIC" : - (queue->param.sched.sync == ODP_SCHED_SYNC_ORDERED ? - "ODP_SCHED_SYNC_ORDERED" : "unknown"))); + _ODP_PRINT(" sync %s\n", + queue->param.sched.sync == ODP_SCHED_SYNC_PARALLEL ? + "ODP_SCHED_SYNC_PARALLEL" : + (queue->param.sched.sync == ODP_SCHED_SYNC_ATOMIC ? + "ODP_SCHED_SYNC_ATOMIC" : + (queue->param.sched.sync == ODP_SCHED_SYNC_ORDERED ? + "ODP_SCHED_SYNC_ORDERED" : "unknown"))); prio = queue->param.sched.prio; - ODP_PRINT(" priority %i (%i in API)\n", max_prio - prio, prio); - ODP_PRINT(" group %i\n", queue->param.sched.group); + _ODP_PRINT(" priority %i (%i in API)\n", max_prio - prio, prio); + _ODP_PRINT(" group %i\n", queue->param.sched.group); if (_odp_sched_id == _ODP_SCHED_ID_BASIC) - ODP_PRINT(" spread %i\n", _odp_sched_basic_get_spread(queue_id)); + _ODP_PRINT(" spread %i\n", _odp_sched_basic_get_spread(queue_id)); } if (queue->pktin.pktio != ODP_PKTIO_INVALID) { if (!odp_pktio_info(queue->pktin.pktio, &pktio_info)) - ODP_PRINT(" pktin %s\n", pktio_info.name); + _ODP_PRINT(" pktin %s\n", pktio_info.name); } if (queue->pktout.pktio != ODP_PKTIO_INVALID) { if (!odp_pktio_info(queue->pktout.pktio, &pktio_info)) - ODP_PRINT(" pktout %s\n", pktio_info.name); + _ODP_PRINT(" pktout %s\n", pktio_info.name); } - ODP_PRINT(" timers %" PRIu64 "\n", - odp_atomic_load_u64(&queue->num_timers)); - ODP_PRINT(" status %s\n", - queue->status == QUEUE_STATUS_READY ? "ready" : - (queue->status == QUEUE_STATUS_NOTSCHED ? "not scheduled" : - (queue->status == QUEUE_STATUS_SCHED ? "scheduled" : "unknown"))); - ODP_PRINT(" param.size %" PRIu32 "\n", queue->param.size); + _ODP_PRINT(" timers %" PRIu64 "\n", odp_atomic_load_u64(&queue->num_timers)); + _ODP_PRINT(" status %s\n", + queue->status == QUEUE_STATUS_READY ? "ready" : + (queue->status == QUEUE_STATUS_NOTSCHED ? "not scheduled" : + (queue->status == QUEUE_STATUS_SCHED ? "scheduled" : "unknown"))); + _ODP_PRINT(" param.size %" PRIu32 "\n", queue->param.size); if (queue->queue_lf) { - ODP_PRINT(" implementation queue_lf\n"); - ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", - _odp_queue_lf_length(queue->queue_lf), _odp_queue_lf_max_length()); + _ODP_PRINT(" implementation queue_lf\n"); + _ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", + _odp_queue_lf_length(queue->queue_lf), _odp_queue_lf_max_length()); } else if (queue->spsc) { - ODP_PRINT(" implementation ring_spsc\n"); - ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", - ring_spsc_length(&queue->ring_spsc), queue->ring_mask + 1); + _ODP_PRINT(" implementation ring_spsc\n"); + _ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", + ring_spsc_length(&queue->ring_spsc), queue->ring_mask + 1); } else if (queue->type == ODP_QUEUE_TYPE_SCHED) { - ODP_PRINT(" implementation ring_st\n"); - ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", - ring_st_length(&queue->ring_st), queue->ring_mask + 1); + _ODP_PRINT(" implementation ring_st\n"); + _ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", + ring_st_length(&queue->ring_st), queue->ring_mask + 1); } else { - ODP_PRINT(" implementation ring_mpmc\n"); - ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", - ring_mpmc_length(&queue->ring_mpmc), queue->ring_mask + 1); + _ODP_PRINT(" implementation ring_mpmc\n"); + _ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", + ring_mpmc_length(&queue->ring_mpmc), queue->ring_mask + 1); } - ODP_PRINT("\n"); + _ODP_PRINT("\n"); UNLOCK(queue); } @@ -784,13 +777,13 @@ static void queue_print_all(void) odp_schedule_sync_t sync = ODP_SCHED_SYNC_PARALLEL; odp_schedule_group_t grp = ODP_SCHED_GROUP_INVALID; - ODP_PRINT("\nList of all queues\n"); - ODP_PRINT("------------------\n"); - ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio grp", col_width, "name"); + _ODP_PRINT("\nList of all queues\n"); + _ODP_PRINT("------------------\n"); + _ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio grp", col_width, "name"); if (_odp_sched_id == _ODP_SCHED_ID_BASIC) - ODP_PRINT(" spr\n"); + _ODP_PRINT(" spr\n"); else - ODP_PRINT("\n"); + _ODP_PRINT("\n"); for (i = 0; i < CONFIG_MAX_QUEUES; i++) { queue_entry_t *queue = qentry_from_index(i); @@ -849,9 +842,9 @@ static void queue_print_all(void) order_c = (order == ODP_QUEUE_ORDER_KEEP) ? 'K' : 'I'; - ODP_PRINT("%4u %-*s %c %2s %2s", index, col_width, name, type_c, - status_str, bl_str); - ODP_PRINT(" %c %c %c %6u %6u", enq_c, deq_c, order_c, len, max_len); + _ODP_PRINT("%4u %-*s %c %2s %2s", index, col_width, name, type_c, + status_str, bl_str); + _ODP_PRINT(" %c %c %c %6u %6u", enq_c, deq_c, order_c, len, max_len); if (type == ODP_QUEUE_TYPE_SCHED) { sync_c = (sync == ODP_SCHED_SYNC_PARALLEL) ? 'P' : @@ -859,16 +852,16 @@ static void queue_print_all(void) /* Print prio level matching odp_schedule_print() output */ prio = odp_schedule_max_prio() - prio; - ODP_PRINT(" %c %4i %3i", sync_c, prio, grp); + _ODP_PRINT(" %c %4i %3i", sync_c, prio, grp); if (_odp_sched_id == _ODP_SCHED_ID_BASIC) - ODP_PRINT(" %3i", spr); + _ODP_PRINT(" %3i", spr); } - ODP_PRINT("\n"); + _ODP_PRINT("\n"); } - ODP_PRINT("\n"); + _ODP_PRINT("\n"); } static inline int _sched_queue_enq_multi(odp_queue_t handle, @@ -908,7 +901,7 @@ static inline int _sched_queue_enq_multi(odp_queue_t handle, /* Add queue to scheduling */ if (sched && _odp_sched_fn->sched_queue(queue->index)) - ODP_ABORT("schedule_queue failed\n"); + _ODP_ABORT("schedule_queue failed\n"); return num_enq; } @@ -1043,7 +1036,7 @@ static int queue_init(queue_entry_t *queue, const char *name, queue_size = _ODP_ROUNDUP_POWER2_U32(queue_size); if (queue_size > _odp_queue_glb->config.max_queue_size) { - ODP_ERR("Too large queue size %u\n", queue_size); + _ODP_ERR("Too large queue size %u\n", queue_size); return -1; } diff --git a/platform/linux-generic/odp_queue_if.c b/platform/linux-generic/odp_queue_if.c index 9ebd5db86..b702e0105 100644 --- a/platform/linux-generic/odp_queue_if.c +++ b/platform/linux-generic/odp_queue_if.c @@ -120,7 +120,7 @@ int _odp_queue_init_global(void) _odp_queue_fn = &_odp_queue_scalable_fn; _odp_queue_api = &_odp_queue_scalable_api; } else { - ODP_ABORT("Unknown scheduler specified via ODP_SCHEDULER\n"); + _ODP_ABORT("Unknown scheduler specified via ODP_SCHEDULER\n"); return -1; } diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c index f511bcec5..9004402b8 100644 --- a/platform/linux-generic/odp_queue_lf.c +++ b/platform/linux-generic/odp_queue_lf.c @@ -264,8 +264,8 @@ uint32_t _odp_queue_lf_init_global(uint32_t *queue_lf_size, /* 16 byte lockfree CAS operation is needed. */ lockfree = lockfree_check_u128(); - ODP_DBG("\nLock-free queue init\n"); - ODP_DBG(" u128 lock-free: %i\n\n", lockfree); + _ODP_DBG("\nLock-free queue init\n"); + _ODP_DBG(" u128 lock-free: %i\n\n", lockfree); if (!lockfree) return 0; @@ -302,7 +302,7 @@ void _odp_queue_lf_term_global(void) shm = queue_lf_glb->shm; if (odp_shm_free(shm) < 0) - ODP_ERR("shm free failed"); + _ODP_ERR("shm free failed"); } static void init_queue(queue_lf_t *queue_lf) @@ -321,7 +321,7 @@ void *_odp_queue_lf_create(queue_entry_t *queue) queue_lf_t *queue_lf = NULL; if (queue_lf_glb == NULL) { - ODP_ERR("No lock-free queues available\n"); + _ODP_ERR("No lock-free queues available\n"); return NULL; } diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index 049e7dae8..4a4be1ecf 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -170,7 +170,7 @@ static int queue_init(queue_entry_t *queue, const char *name, _odp_rwin_alloc(queue_shm_pool, queue->param.sched.lock_count); if (sched_elem->rwin == NULL) { - ODP_ERR("Reorder window not created\n"); + _ODP_ERR("Reorder window not created\n"); goto rwin_create_failed; } } @@ -178,7 +178,7 @@ static int queue_init(queue_entry_t *queue, const char *name, sched_elem->sched_prio = prio; sched_elem->schedq = _odp_sched_queue_add(param->sched.group, prio); - ODP_ASSERT(sched_elem->schedq != NULL); + _ODP_ASSERT(sched_elem->schedq != NULL); } return 0; @@ -196,7 +196,7 @@ static int queue_init_global(void) uint64_t min_alloc; uint64_t max_alloc; - ODP_DBG("Queue init ... "); + _ODP_DBG("Queue init ... "); /* Fill in queue entry field offsets for inline functions */ memset(&_odp_queue_inline_offset, 0, @@ -222,7 +222,7 @@ static int queue_init_global(void) pool_size, min_alloc, max_alloc, 0); if (queue_shm_pool == NULL) { - ODP_ERR("Failed to allocate shared memory pool for" + _ODP_ERR("Failed to allocate shared memory pool for" " queues\n"); goto queue_shm_pool_create_failed; } @@ -231,7 +231,7 @@ static int queue_init_global(void) shm_pool_alloc_align(queue_shm_pool, sizeof(queue_table_t)); if (queue_tbl == NULL) { - ODP_ERR("Failed to reserve shared memory for queue table\n"); + _ODP_ERR("Failed to reserve shared memory for queue table\n"); goto queue_tbl_ishm_alloc_failed; } @@ -247,13 +247,11 @@ static int queue_init_global(void) queue->handle = (odp_queue_t)queue; } - ODP_DBG("done\n"); - ODP_DBG("Queue init global\n"); - ODP_DBG(" struct queue_entry_s size %zu\n", - sizeof(struct queue_entry_s)); - ODP_DBG(" queue_entry_t size %zu\n", - sizeof(queue_entry_t)); - ODP_DBG("\n"); + _ODP_DBG("done\n"); + _ODP_DBG("Queue init global\n"); + _ODP_DBG(" struct queue_entry_s size %zu\n", sizeof(struct queue_entry_s)); + _ODP_DBG(" queue_entry_t size %zu\n", sizeof(queue_entry_t)); + _ODP_DBG("\n"); return 0; @@ -276,7 +274,7 @@ static int queue_term_global(void) queue = &queue_tbl->queue[i]; if (__atomic_load_n(&queue->status, __ATOMIC_RELAXED) != QUEUE_STATUS_FREE) { - ODP_ERR("Not destroyed queue: %s\n", queue->name); + _ODP_ERR("Not destroyed queue: %s\n", queue->name); rc = -1; } } @@ -285,7 +283,7 @@ static int queue_term_global(void) ret = _odp_ishm_pool_destroy(queue_shm_pool); if (ret < 0) { - ODP_ERR("Failed to destroy shared memory pool for queues\n"); + _ODP_ERR("Failed to destroy shared memory pool for queues\n"); rc = -1; } @@ -363,7 +361,7 @@ static odp_queue_t queue_create(const char *name, if (type == ODP_QUEUE_TYPE_SCHED) { if (param->sched.prio < odp_schedule_min_prio() || param->sched.prio > odp_schedule_max_prio()) { - ODP_ERR("Bad queue priority: %i\n", param->sched.prio); + _ODP_ERR("Bad queue priority: %i\n", param->sched.prio); return ODP_QUEUE_INVALID; } } @@ -467,7 +465,7 @@ static int queue_destroy(odp_queue_t handle) if (q->rwin != NULL) { if (_odp_rwin_free(queue_shm_pool, q->rwin) < 0) { - ODP_ERR("Failed to free reorder window\n"); + _ODP_ERR("Failed to free reorder window\n"); UNLOCK(&queue->lock); return -1; } @@ -897,15 +895,14 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info) int status; if (odp_unlikely(info == NULL)) { - ODP_ERR("Unable to store info, NULL ptr given\n"); + _ODP_ERR("Unable to store info, NULL ptr given\n"); return -1; } queue_id = queue_to_id(handle); if (odp_unlikely(queue_id >= CONFIG_MAX_QUEUES)) { - ODP_ERR("Invalid queue handle:%" PRIu64 "\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Invalid queue handle:%" PRIu64 "\n", odp_queue_to_u64(handle)); return -1; } @@ -917,7 +914,7 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info) if (odp_unlikely(status == QUEUE_STATUS_FREE || status == QUEUE_STATUS_DESTROYED)) { UNLOCK(&queue->lock); - ODP_ERR("Invalid queue status:%d\n", status); + _ODP_ERR("Invalid queue status:%d\n", status); return -1; } @@ -939,8 +936,7 @@ static void queue_print(odp_queue_t handle) queue_id = queue_to_id(handle); if (odp_unlikely(queue_id >= CONFIG_MAX_QUEUES)) { - ODP_ERR("Invalid queue handle: 0x%" PRIx64 "\n", - odp_queue_to_u64(handle)); + _ODP_ERR("Invalid queue handle: 0x%" PRIx64 "\n", odp_queue_to_u64(handle)); return; } @@ -952,50 +948,49 @@ static void queue_print(odp_queue_t handle) if (odp_unlikely(status == QUEUE_STATUS_FREE || status == QUEUE_STATUS_DESTROYED)) { UNLOCK(&queue->lock); - ODP_ERR("Invalid queue status:%d\n", status); + _ODP_ERR("Invalid queue status:%d\n", status); return; } - ODP_PRINT("\nQueue info\n"); - ODP_PRINT("----------\n"); - ODP_PRINT(" handle %p\n", (void *)queue->handle); - ODP_PRINT(" index %" PRIu32 "\n", queue->index); - ODP_PRINT(" name %s\n", queue->name); - ODP_PRINT(" enq mode %s\n", - queue->param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : - (queue->param.enq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : - (queue->param.enq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : - "unknown"))); - ODP_PRINT(" deq mode %s\n", - queue->param.deq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : - (queue->param.deq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : - (queue->param.deq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : - "unknown"))); - ODP_PRINT(" type %s\n", - queue->type == ODP_QUEUE_TYPE_PLAIN ? "ODP_QUEUE_TYPE_PLAIN" : - (queue->type == ODP_QUEUE_TYPE_SCHED ? "ODP_QUEUE_TYPE_SCHED" : "unknown")); + _ODP_PRINT("\nQueue info\n"); + _ODP_PRINT("----------\n"); + _ODP_PRINT(" handle %p\n", (void *)queue->handle); + _ODP_PRINT(" index %" PRIu32 "\n", queue->index); + _ODP_PRINT(" name %s\n", queue->name); + _ODP_PRINT(" enq mode %s\n", + queue->param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : + (queue->param.enq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : + (queue->param.enq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : + "unknown"))); + _ODP_PRINT(" deq mode %s\n", + queue->param.deq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : + (queue->param.deq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : + (queue->param.deq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : + "unknown"))); + _ODP_PRINT(" type %s\n", + queue->type == ODP_QUEUE_TYPE_PLAIN ? "ODP_QUEUE_TYPE_PLAIN" : + (queue->type == ODP_QUEUE_TYPE_SCHED ? "ODP_QUEUE_TYPE_SCHED" : "unknown")); if (queue->type == ODP_QUEUE_TYPE_SCHED) { - ODP_PRINT(" sync %s\n", - queue->param.sched.sync == ODP_SCHED_SYNC_PARALLEL ? - "ODP_SCHED_SYNC_PARALLEL" : - (queue->param.sched.sync == ODP_SCHED_SYNC_ATOMIC ? - "ODP_SCHED_SYNC_ATOMIC" : - (queue->param.sched.sync == ODP_SCHED_SYNC_ORDERED ? - "ODP_SCHED_SYNC_ORDERED" : "unknown"))); - ODP_PRINT(" priority %d\n", queue->param.sched.prio); - ODP_PRINT(" group %d\n", queue->param.sched.group); + _ODP_PRINT(" sync %s\n", + queue->param.sched.sync == ODP_SCHED_SYNC_PARALLEL ? + "ODP_SCHED_SYNC_PARALLEL" : + (queue->param.sched.sync == ODP_SCHED_SYNC_ATOMIC ? + "ODP_SCHED_SYNC_ATOMIC" : + (queue->param.sched.sync == ODP_SCHED_SYNC_ORDERED ? + "ODP_SCHED_SYNC_ORDERED" : "unknown"))); + _ODP_PRINT(" priority %d\n", queue->param.sched.prio); + _ODP_PRINT(" group %d\n", queue->param.sched.group); } if (queue->pktin.pktio != ODP_PKTIO_INVALID) { if (!odp_pktio_info(queue->pktin.pktio, &pktio_info)) - ODP_PRINT(" pktin %s\n", pktio_info.name); + _ODP_PRINT(" pktin %s\n", pktio_info.name); } if (queue->pktout.pktio != ODP_PKTIO_INVALID) { if (!odp_pktio_info(queue->pktout.pktio, &pktio_info)) - ODP_PRINT(" pktout %s\n", pktio_info.name); + _ODP_PRINT(" pktout %s\n", pktio_info.name); } - ODP_PRINT(" timers %" PRIu64 "\n", - odp_atomic_load_u64(&queue->num_timers)); - ODP_PRINT(" param.size %" PRIu32 "\n", queue->param.size); - ODP_PRINT("\n"); + _ODP_PRINT(" timers %" PRIu64 "\n", odp_atomic_load_u64(&queue->num_timers)); + _ODP_PRINT(" param.size %" PRIu32 "\n", queue->param.size); + _ODP_PRINT("\n"); UNLOCK(&queue->lock); } @@ -1016,9 +1011,9 @@ static void queue_print_all(void) char type_c, enq_c, deq_c, order_c, sync_c; const int col_width = 24; - ODP_PRINT("\nList of all queues\n"); - ODP_PRINT("------------------\n"); - ODP_PRINT(" idx %-*s type blk enq deq ord sync prio\n", col_width, "name"); + _ODP_PRINT("\nList of all queues\n"); + _ODP_PRINT("------------------\n"); + _ODP_PRINT(" idx %-*s type blk enq deq ord sync prio\n", col_width, "name"); for (i = 0; i < CONFIG_MAX_QUEUES; i++) { queue_entry_t *queue = &queue_tbl->queue[i]; @@ -1057,19 +1052,19 @@ static void queue_print_all(void) order_c = (order == ODP_QUEUE_ORDER_KEEP) ? 'K' : 'I'; - ODP_PRINT("%4u %-*s %c %2s", index, col_width, name, type_c, bl_str); - ODP_PRINT(" %c %c %c", enq_c, deq_c, order_c); + _ODP_PRINT("%4u %-*s %c %2s", index, col_width, name, type_c, bl_str); + _ODP_PRINT(" %c %c %c", enq_c, deq_c, order_c); if (type == ODP_QUEUE_TYPE_SCHED) { sync_c = (sync == ODP_SCHED_SYNC_PARALLEL) ? 'P' : ((sync == ODP_SCHED_SYNC_ATOMIC) ? 'A' : 'O'); - ODP_PRINT(" %c %4i", sync_c, prio); + _ODP_PRINT(" %c %4i", sync_c, prio); } - ODP_PRINT("\n"); + _ODP_PRINT("\n"); } - ODP_PRINT("\n"); + _ODP_PRINT("\n"); } static uint64_t queue_to_u64(odp_queue_t hdl) diff --git a/platform/linux-generic/odp_queue_spsc.c b/platform/linux-generic/odp_queue_spsc.c index b2fed7d0d..74cc740e0 100644 --- a/platform/linux-generic/odp_queue_spsc.c +++ b/platform/linux-generic/odp_queue_spsc.c @@ -46,7 +46,7 @@ static inline int spsc_enq_multi(odp_queue_t handle, event_index_from_hdr(buf_idx, event_hdr, num); if (odp_unlikely(queue->status < QUEUE_STATUS_READY)) { - ODP_ERR("Bad queue status\n"); + _ODP_ERR("Bad queue status\n"); return -1; } diff --git a/platform/linux-generic/odp_rwlock.c b/platform/linux-generic/odp_rwlock.c deleted file mode 100644 index 03af7d26c..000000000 --- a/platform/linux-generic/odp_rwlock.c +++ /dev/null @@ -1,83 +0,0 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <stdbool.h> -#include <odp/api/atomic.h> -#include <odp/api/rwlock.h> -#include <odp/api/cpu.h> - -#include <odp/api/plat/atomic_inlines.h> -#include <odp/api/plat/cpu_inlines.h> - -void odp_rwlock_init(odp_rwlock_t *rwlock) -{ - odp_atomic_init_u32(&rwlock->cnt, 0); -} - -void odp_rwlock_read_lock(odp_rwlock_t *rwlock) -{ - uint32_t cnt; - int is_locked = 0; - - while (is_locked == 0) { - cnt = odp_atomic_load_u32(&rwlock->cnt); - /* waiting for read lock */ - if ((int32_t)cnt < 0) { - odp_cpu_pause(); - continue; - } - is_locked = odp_atomic_cas_acq_u32(&rwlock->cnt, - &cnt, cnt + 1); - } -} - -int odp_rwlock_read_trylock(odp_rwlock_t *rwlock) -{ - uint32_t cnt = odp_atomic_load_u32(&rwlock->cnt); - - while (cnt != (uint32_t)-1) { - if (odp_atomic_cas_acq_u32(&rwlock->cnt, &cnt, cnt + 1)) - return 1; - } - - return 0; -} - -void odp_rwlock_read_unlock(odp_rwlock_t *rwlock) -{ - odp_atomic_sub_rel_u32(&rwlock->cnt, 1); -} - -void odp_rwlock_write_lock(odp_rwlock_t *rwlock) -{ - uint32_t cnt; - int is_locked = 0; - - while (is_locked == 0) { - uint32_t zero = 0; - - cnt = odp_atomic_load_u32(&rwlock->cnt); - /* lock acquired, wait */ - if (cnt != 0) { - odp_cpu_pause(); - continue; - } - is_locked = odp_atomic_cas_acq_u32(&rwlock->cnt, - &zero, (uint32_t)-1); - } -} - -int odp_rwlock_write_trylock(odp_rwlock_t *rwlock) -{ - uint32_t zero = 0; - - return odp_atomic_cas_acq_u32(&rwlock->cnt, &zero, (uint32_t)-1); -} - -void odp_rwlock_write_unlock(odp_rwlock_t *rwlock) -{ - odp_atomic_store_rel_u32(&rwlock->cnt, 0); -} diff --git a/platform/linux-generic/odp_rwlock_api.c b/platform/linux-generic/odp_rwlock_api.c new file mode 100644 index 000000000..217479598 --- /dev/null +++ b/platform/linux-generic/odp_rwlock_api.c @@ -0,0 +1,10 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/rwlock.h> + +#define _ODP_NO_INLINE +#include <odp/api/plat/rwlock_inlines.h> diff --git a/platform/linux-generic/odp_rwlock_recursive.c b/platform/linux-generic/odp_rwlock_recursive.c deleted file mode 100644 index d7bbf3c8b..000000000 --- a/platform/linux-generic/odp_rwlock_recursive.c +++ /dev/null @@ -1,108 +0,0 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp/api/rwlock_recursive.h> -#include <odp/api/thread.h> -#include <odp/api/plat/thread_inlines.h> -#include <string.h> - -#define NO_OWNER (-1) - -void odp_rwlock_recursive_init(odp_rwlock_recursive_t *rlock) -{ - memset(rlock, 0, sizeof(odp_rwlock_recursive_t)); - odp_rwlock_init(&rlock->lock); - rlock->wr_owner = NO_OWNER; -} - -/* Multiple readers can recurse the lock concurrently */ -void odp_rwlock_recursive_read_lock(odp_rwlock_recursive_t *rlock) -{ - int thr = odp_thread_id(); - - if (rlock->rd_cnt[thr]) { - rlock->rd_cnt[thr]++; - return; - } - - odp_rwlock_read_lock(&rlock->lock); - rlock->rd_cnt[thr] = 1; -} - -/* Multiple readers can recurse the lock concurrently */ -int odp_rwlock_recursive_read_trylock(odp_rwlock_recursive_t *rlock) -{ - int thr = odp_thread_id(); - - if (rlock->rd_cnt[thr]) { - rlock->rd_cnt[thr]++; - return 1; - } - - if (odp_rwlock_read_trylock(&rlock->lock)) { - rlock->rd_cnt[thr] = 1; - return 1; - } - - return 0; -} - -void odp_rwlock_recursive_read_unlock(odp_rwlock_recursive_t *rlock) -{ - int thr = odp_thread_id(); - - rlock->rd_cnt[thr]--; - - if (rlock->rd_cnt[thr] > 0) - return; - - odp_rwlock_read_unlock(&rlock->lock); -} - -/* Only one writer can recurse the lock */ -void odp_rwlock_recursive_write_lock(odp_rwlock_recursive_t *rlock) -{ - int thr = odp_thread_id(); - - if (rlock->wr_owner == thr) { - rlock->wr_cnt++; - return; - } - - odp_rwlock_write_lock(&rlock->lock); - rlock->wr_owner = thr; - rlock->wr_cnt = 1; -} - -/* Only one writer can recurse the lock */ -int odp_rwlock_recursive_write_trylock(odp_rwlock_recursive_t *rlock) -{ - int thr = odp_thread_id(); - - if (rlock->wr_owner == thr) { - rlock->wr_cnt++; - return 1; - } - - if (odp_rwlock_write_trylock(&rlock->lock)) { - rlock->wr_owner = thr; - rlock->wr_cnt = 1; - return 1; - } - - return 0; -} - -void odp_rwlock_recursive_write_unlock(odp_rwlock_recursive_t *rlock) -{ - rlock->wr_cnt--; - - if (rlock->wr_cnt > 0) - return; - - rlock->wr_owner = NO_OWNER; - odp_rwlock_write_unlock(&rlock->lock); -} diff --git a/platform/linux-generic/odp_rwlock_recursive_api.c b/platform/linux-generic/odp_rwlock_recursive_api.c new file mode 100644 index 000000000..b2580ae42 --- /dev/null +++ b/platform/linux-generic/odp_rwlock_recursive_api.c @@ -0,0 +1,10 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/rwlock_recursive.h> + +#define _ODP_NO_INLINE +#include <odp/api/plat/rwlock_recursive_inlines.h> diff --git a/platform/linux-generic/odp_schedule_api.c b/platform/linux-generic/odp_schedule_api.c new file mode 100644 index 000000000..4be1ba6f7 --- /dev/null +++ b/platform/linux-generic/odp_schedule_api.c @@ -0,0 +1,11 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/schedule.h> + +/* Non-inlined functions for ABI compat mode */ +#define _ODP_NO_INLINE +#include <odp/api/plat/schedule_inlines.h> diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 112c392ec..594360326 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -34,6 +34,7 @@ #include <odp_queue_basic_internal.h> #include <odp_libconfig_internal.h> #include <odp/api/plat/queue_inlines.h> +#include <odp/api/plat/schedule_inline_types.h> #include <odp_global_data.h> #include <odp_event_internal.h> #include <odp_macros_internal.h> @@ -47,8 +48,11 @@ /* Number of priority levels */ #define NUM_PRIO 8 -/* Number of scheduling groups */ -#define NUM_SCHED_GRPS 32 +/* Group mask (prio_grp_mask) size in bits */ +#define GRP_MASK_BITS 64 + +/* Number of scheduling groups. Maximum value is GRP_MASK_BITS. */ +#define NUM_SCHED_GRPS GRP_MASK_BITS /* Spread balancing frequency. Balance every BALANCE_ROUNDS_M1 + 1 scheduling rounds. */ #define BALANCE_ROUNDS_M1 0xfffff @@ -184,8 +188,8 @@ typedef struct ODP_ALIGNED_CACHE { odp_event_t ev[STASH_SIZE]; } stash; + uint64_t grp_mask; uint32_t grp_epoch; - uint32_t grp_mask; uint16_t num_grp; uint8_t grp_idx; uint8_t grp[NUM_SCHED_GRPS]; @@ -247,7 +251,7 @@ typedef struct { prio_q_mask_t prio_q_mask[NUM_SCHED_GRPS][NUM_PRIO]; /* Groups on a priority level that have queues created */ - odp_atomic_u32_t prio_grp_mask[NUM_PRIO]; + odp_atomic_u64_t prio_grp_mask[NUM_PRIO]; struct { uint8_t grp; @@ -296,7 +300,7 @@ typedef struct { } sched_global_t; /* Check that queue[] variables are large enough */ -ODP_STATIC_ASSERT(NUM_SCHED_GRPS <= 32, "Group mask is 32 bits"); +ODP_STATIC_ASSERT(NUM_SCHED_GRPS <= GRP_MASK_BITS, "Groups do not fit into group mask"); ODP_STATIC_ASSERT(NUM_PRIO <= 256, "Prio_does_not_fit_8_bits"); ODP_STATIC_ASSERT(MAX_SPREAD <= 256, "Spread_does_not_fit_8_bits"); ODP_STATIC_ASSERT(CONFIG_QUEUE_MAX_ORD_LOCKS <= 256, @@ -309,6 +313,40 @@ static sched_global_t *sched; /* Thread local scheduler context */ static __thread sched_local_t sched_local; +static void prio_grp_mask_init(void) +{ + int i; + + for (i = 0; i < NUM_PRIO; i++) + odp_atomic_init_u64(&sched->prio_grp_mask[i], 0); +} + +static inline void prio_grp_mask_set(int prio, int grp) +{ + uint64_t grp_mask = 0x1u << grp; + uint64_t mask = odp_atomic_load_u64(&sched->prio_grp_mask[prio]); + + odp_atomic_store_u64(&sched->prio_grp_mask[prio], mask | grp_mask); + + sched->prio_grp_count[prio][grp]++; +} + +static inline void prio_grp_mask_clear(int prio, int grp) +{ + uint64_t grp_mask = 0x1u << grp; + uint64_t mask = odp_atomic_load_u64(&sched->prio_grp_mask[prio]); + + sched->prio_grp_count[prio][grp]--; + + if (sched->prio_grp_count[prio][grp] == 0) + odp_atomic_store_u64(&sched->prio_grp_mask[prio], mask &= (~grp_mask)); +} + +static inline uint64_t prio_grp_mask_check(int prio, uint64_t grp_mask) +{ + return odp_atomic_load_u64(&sched->prio_grp_mask[prio]) & grp_mask; +} + static int read_burst_size_conf(uint8_t out_tbl[], const char *conf_str, int min_val, int max_val, int print) { @@ -320,7 +358,7 @@ static int read_burst_size_conf(uint8_t out_tbl[], const char *conf_str, if (_odp_libconfig_lookup_array(conf_str, burst_val, NUM_PRIO) != NUM_PRIO) { - ODP_ERR("Config option '%s' not found.\n", conf_str); + _ODP_ERR("Config option '%s' not found.\n", conf_str); return -1; } @@ -333,7 +371,7 @@ static int read_burst_size_conf(uint8_t out_tbl[], const char *conf_str, int val = burst_val[i]; if (val > max_val || val < min_val) { - ODP_ERR("Bad value for %s: %i\n", conf_str, val); + _ODP_ERR("Bad value for %s: %i\n", conf_str, val); return -1; } len += snprintf(&line[len], n - len, " %3i", val); @@ -341,7 +379,7 @@ static int read_burst_size_conf(uint8_t out_tbl[], const char *conf_str, out_tbl[i] = val; } if (print) - ODP_PRINT("%s\n", line); + _ODP_PRINT("%s\n", line); return 0; } @@ -351,49 +389,49 @@ static int read_config_file(sched_global_t *sched) const char *str; int val = 0; - ODP_PRINT("Scheduler config:\n"); + _ODP_PRINT("Scheduler config:\n"); str = "sched_basic.prio_spread"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } if (val > MAX_SPREAD || val < MIN_SPREAD) { - ODP_ERR("Bad value %s = %u [min: %u, max: %u]\n", str, val, - MIN_SPREAD, MAX_SPREAD); + _ODP_ERR("Bad value %s = %u [min: %u, max: %u]\n", str, val, + MIN_SPREAD, MAX_SPREAD); return -1; } sched->config.num_spread = val; - ODP_PRINT(" %s: %i\n", str, val); + _ODP_PRINT(" %s: %i\n", str, val); str = "sched_basic.prio_spread_weight"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } if (val > MAX_PREFER_WEIGHT || val < MIN_PREFER_WEIGHT) { - ODP_ERR("Bad value %s = %u [min: %u, max: %u]\n", str, val, - MIN_PREFER_WEIGHT, MAX_PREFER_WEIGHT); + _ODP_ERR("Bad value %s = %u [min: %u, max: %u]\n", str, val, + MIN_PREFER_WEIGHT, MAX_PREFER_WEIGHT); return -1; } sched->config.prefer_ratio = val + 1; - ODP_PRINT(" %s: %i\n", str, val); + _ODP_PRINT(" %s: %i\n", str, val); str = "sched_basic.load_balance"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } if (val > 1 || val < 0) { - ODP_ERR("Bad value %s = %i\n", str, val); + _ODP_ERR("Bad value %s = %i\n", str, val); return -1; } - ODP_PRINT(" %s: %i\n", str, val); + _ODP_PRINT(" %s: %i\n", str, val); sched->load_balance = 1; if (val == 0 || sched->config.num_spread == 1) @@ -444,34 +482,34 @@ static int read_config_file(sched_global_t *sched) str = "sched_basic.group_enable.all"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } sched->config_if.group_enable.all = val; - ODP_PRINT(" %s: %i\n", str, val); + _ODP_PRINT(" %s: %i\n", str, val); str = "sched_basic.group_enable.worker"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } sched->config_if.group_enable.worker = val; - ODP_PRINT(" %s: %i\n", str, val); + _ODP_PRINT(" %s: %i\n", str, val); str = "sched_basic.group_enable.control"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } sched->config_if.group_enable.control = val; - ODP_PRINT(" %s: %i\n", str, val); + _ODP_PRINT(" %s: %i\n", str, val); - ODP_PRINT(" dynamic load balance: %s\n", sched->load_balance ? "ON" : "OFF"); + _ODP_PRINT(" dynamic load balance: %s\n", sched->load_balance ? "ON" : "OFF"); - ODP_PRINT("\n"); + _ODP_PRINT("\n"); return 0; } @@ -517,14 +555,14 @@ static int schedule_init_global(void) int prefer_ratio; uint32_t ring_size, num_rings; - ODP_DBG("Schedule init ... "); + _ODP_DBG("Schedule init ... "); shm = odp_shm_reserve("_odp_sched_basic_global", sizeof(sched_global_t), ODP_CACHE_LINE_SIZE, 0); if (shm == ODP_SHM_INVALID) { - ODP_ERR("Schedule init: Shm reserve failed.\n"); + _ODP_ERR("Schedule init: Shm reserve failed.\n"); return -1; } @@ -553,7 +591,7 @@ static int schedule_init_global(void) } ring_size = _ODP_ROUNDUP_POWER2_U32(ring_size); - ODP_ASSERT(ring_size <= MAX_RING_SIZE); + _ODP_ASSERT(ring_size <= MAX_RING_SIZE); sched->ring_mask = ring_size - 1; /* Each ring can hold in maximum ring_size-1 queues. Due to ring size round up, @@ -583,8 +621,7 @@ static int schedule_init_global(void) odp_atomic_init_u32(&sched->grp_epoch, 0); odp_atomic_init_u32(&sched->next_rand, 0); - for (i = 0; i < NUM_PRIO; i++) - odp_atomic_init_u32(&sched->prio_grp_mask[i], 0); + prio_grp_mask_init(); for (i = 0; i < NUM_SCHED_GRPS; i++) { memset(sched->sched_grp[i].name, 0, ODP_SCHED_GROUP_NAME_LEN); @@ -604,7 +641,7 @@ static int schedule_init_global(void) odp_thrmask_setall(&sched->mask_all); - ODP_DBG("done\n"); + _ODP_DBG("done\n"); return 0; } @@ -631,7 +668,7 @@ static int schedule_term_global(void) num = _odp_sched_queue_deq(qi, events, 1, 1); if (num > 0) - ODP_ERR("Queue not empty\n"); + _ODP_ERR("Queue not empty\n"); } } } @@ -639,7 +676,7 @@ static int schedule_term_global(void) ret = odp_shm_free(sched->shm); if (ret < 0) { - ODP_ERR("Shm free failed for odp_scheduler"); + _ODP_ERR("Shm free failed for odp_scheduler"); rc = -1; } @@ -663,7 +700,7 @@ static inline int grp_update_tbl(void) int i; int num = 0; int thr = sched_local.thr; - uint32_t mask = 0; + uint64_t mask = 0; odp_ticketlock_lock(&sched->grp_lock); @@ -687,32 +724,6 @@ static inline int grp_update_tbl(void) return num; } -static inline void prio_grp_mask_set(int prio, int grp) -{ - uint32_t grp_mask = 0x1u << grp; - uint32_t mask = odp_atomic_load_u32(&sched->prio_grp_mask[prio]); - - odp_atomic_store_u32(&sched->prio_grp_mask[prio], mask | grp_mask); - - sched->prio_grp_count[prio][grp]++; -} - -static inline void prio_grp_mask_clear(int prio, int grp) -{ - uint32_t grp_mask = 0x1u << grp; - uint32_t mask = odp_atomic_load_u32(&sched->prio_grp_mask[prio]); - - sched->prio_grp_count[prio][grp]--; - - if (sched->prio_grp_count[prio][grp] == 0) - odp_atomic_store_u32(&sched->prio_grp_mask[prio], mask &= (~grp_mask)); -} - -static inline uint32_t prio_grp_mask_check(int prio, uint32_t grp_mask) -{ - return odp_atomic_load_u32(&sched->prio_grp_mask[prio]) & grp_mask; -} - static uint32_t schedule_max_ordered_locks(void) { return CONFIG_QUEUE_MAX_ORD_LOCKS; @@ -824,24 +835,24 @@ static int schedule_create_queue(uint32_t queue_index, int prio = prio_level_from_api(sched_param->prio); if (odp_global_rw->schedule_configured == 0) { - ODP_ERR("Scheduler has not been configured\n"); + _ODP_ERR("Scheduler has not been configured\n"); return -1; } if (grp < 0 || grp >= NUM_SCHED_GRPS) { - ODP_ERR("Bad schedule group %i\n", grp); + _ODP_ERR("Bad schedule group %i\n", grp); return -1; } if (grp == ODP_SCHED_GROUP_ALL && !sched->config_if.group_enable.all) { - ODP_ERR("Trying to use disabled ODP_SCHED_GROUP_ALL\n"); + _ODP_ERR("Trying to use disabled ODP_SCHED_GROUP_ALL\n"); return -1; } if (grp == ODP_SCHED_GROUP_CONTROL && !sched->config_if.group_enable.control) { - ODP_ERR("Trying to use disabled ODP_SCHED_GROUP_CONTROL\n"); + _ODP_ERR("Trying to use disabled ODP_SCHED_GROUP_CONTROL\n"); return -1; } if (grp == ODP_SCHED_GROUP_WORKER && !sched->config_if.group_enable.worker) { - ODP_ERR("Trying to use disabled ODP_SCHED_GROUP_WORKER\n"); + _ODP_ERR("Trying to use disabled ODP_SCHED_GROUP_WORKER\n"); return -1; } @@ -849,7 +860,7 @@ static int schedule_create_queue(uint32_t queue_index, if (sched->sched_grp[grp].allocated == 0) { odp_ticketlock_unlock(&sched->grp_lock); - ODP_ERR("Group not created: %i\n", grp); + _ODP_ERR("Group not created: %i\n", grp); return -1; } @@ -901,7 +912,7 @@ static void schedule_destroy_queue(uint32_t queue_index) if ((sched_sync_type(queue_index) == ODP_SCHED_SYNC_ORDERED) && odp_atomic_load_u64(&sched->order[queue_index].ctx) != odp_atomic_load_u64(&sched->order[queue_index].next_ctx)) - ODP_ERR("queue reorder incomplete\n"); + _ODP_ERR("queue reorder incomplete\n"); } static int schedule_sched_queue(uint32_t queue_index) @@ -929,7 +940,7 @@ static void schedule_pktio_start(int pktio_index, int num_pktin, sched->queue[qi].pktio_index = pktio_index; sched->queue[qi].pktin_index = pktin_idx[i]; - ODP_ASSERT(pktin_idx[i] <= MAX_PKTIN_INDEX); + _ODP_ASSERT(pktin_idx[i] <= MAX_PKTIN_INDEX); /* Start polling */ _odp_sched_queue_set_status(qi, QUEUE_STATUS_SCHED); @@ -1001,7 +1012,7 @@ static inline void ordered_stash_release(void) if (odp_unlikely(num_enq < 0)) num_enq = 0; - ODP_DBG("Dropped %i packets\n", num - num_enq); + _ODP_DBG("Dropped %i packets\n", num - num_enq); _odp_event_free_multi(&event_hdr[num_enq], num - num_enq); } } @@ -1048,7 +1059,7 @@ static void schedule_release_ordered(void) static int schedule_term_local(void) { if (sched_local.stash.num_ev) { - ODP_ERR("Locally pre-scheduled events exist.\n"); + _ODP_ERR("Locally pre-scheduled events exist.\n"); return -1; } @@ -1076,7 +1087,7 @@ static void schedule_group_clear(odp_schedule_group_t group) odp_thrmask_zero(&zero); if (group < 0 || group > ODP_SCHED_GROUP_CONTROL) - ODP_ABORT("Invalid scheduling group\n"); + _ODP_ABORT("Invalid scheduling group\n"); grp_update_mask(group, &zero); sched->sched_grp[group].allocated = 0; @@ -1277,7 +1288,7 @@ static inline int poll_pktin(uint32_t qi, int direct_recv, if (odp_unlikely(ret < 0)) num_enq = 0; - ODP_DBG("Dropped %i packets\n", num - num_enq); + _ODP_DBG("Dropped %i packets\n", num - num_enq); _odp_event_free_multi(&b_hdr[num_enq], num - num_enq); } @@ -1463,7 +1474,7 @@ static inline int do_schedule(odp_queue_t *out_q, odp_event_t out_ev[], uint32_t uint32_t sched_round; uint16_t spread_round; uint32_t epoch; - uint32_t my_groups; + uint64_t my_groups; int balance = 0; if (sched_local.stash.num_ev) { @@ -1665,8 +1676,8 @@ static void schedule_order_lock(uint32_t lock_index) queue_index = sched_local.ordered.src_queue; - ODP_ASSERT(lock_index <= sched->queue[queue_index].order_lock_count && - !sched_local.ordered.lock_called.u8[lock_index]); + _ODP_ASSERT(lock_index <= sched->queue[queue_index].order_lock_count && + !sched_local.ordered.lock_called.u8[lock_index]); ord_lock = &sched->order[queue_index].lock[lock_index]; @@ -1694,11 +1705,11 @@ static void schedule_order_unlock(uint32_t lock_index) queue_index = sched_local.ordered.src_queue; - ODP_ASSERT(lock_index <= sched->queue[queue_index].order_lock_count); + _ODP_ASSERT(lock_index <= sched->queue[queue_index].order_lock_count); ord_lock = &sched->order[queue_index].lock[lock_index]; - ODP_ASSERT(sched_local.ordered.ctx == odp_atomic_load_u64(ord_lock)); + _ODP_ASSERT(sched_local.ordered.ctx == odp_atomic_load_u64(ord_lock)); odp_atomic_store_rel_u64(ord_lock, sched_local.ordered.ctx + 1); } @@ -1766,7 +1777,7 @@ static inline int threads_from_mask(int thr_tbl[], int count, const odp_thrmask_ for (i = 0; i < count; i++) { if (thr < 0) { - ODP_ERR("No more threads in the mask\n"); + _ODP_ERR("No more threads in the mask\n"); return -1; } @@ -1785,7 +1796,7 @@ static odp_schedule_group_t schedule_group_create(const char *name, count = odp_thrmask_count(mask); if (count < 0) { - ODP_ERR("Bad thread count\n"); + _ODP_ERR("Bad thread count\n"); return ODP_SCHED_GROUP_INVALID; } @@ -1826,7 +1837,7 @@ static int schedule_group_destroy(odp_schedule_group_t group) int i; if (group >= NUM_SCHED_GRPS || group < SCHED_GROUP_NAMED) { - ODP_ERR("Bad group %i\n", group); + _ODP_ERR("Bad group %i\n", group); return -1; } @@ -1836,7 +1847,7 @@ static int schedule_group_destroy(odp_schedule_group_t group) if (sched->sched_grp[group].allocated == 0) { odp_ticketlock_unlock(&sched->grp_lock); - ODP_ERR("Group not created: %i\n", group); + _ODP_ERR("Group not created: %i\n", group); return -1; } @@ -1876,13 +1887,13 @@ static int schedule_group_join(odp_schedule_group_t group, const odp_thrmask_t * odp_thrmask_t new_mask; if (group >= NUM_SCHED_GRPS || group < SCHED_GROUP_NAMED) { - ODP_ERR("Bad group %i\n", group); + _ODP_ERR("Bad group %i\n", group); return -1; } count = odp_thrmask_count(mask); if (count <= 0) { - ODP_ERR("No threads in the mask\n"); + _ODP_ERR("No threads in the mask\n"); return -1; } @@ -1891,7 +1902,7 @@ static int schedule_group_join(odp_schedule_group_t group, const odp_thrmask_t * thr = odp_thrmask_first(mask); for (i = 0; i < count; i++) { if (thr < 0) { - ODP_ERR("No more threads in the mask\n"); + _ODP_ERR("No more threads in the mask\n"); return -1; } @@ -1903,7 +1914,7 @@ static int schedule_group_join(odp_schedule_group_t group, const odp_thrmask_t * if (sched->sched_grp[group].allocated == 0) { odp_ticketlock_unlock(&sched->grp_lock); - ODP_ERR("Bad group status\n"); + _ODP_ERR("Bad group status\n"); return -1; } @@ -1922,13 +1933,13 @@ static int schedule_group_leave(odp_schedule_group_t group, const odp_thrmask_t odp_thrmask_t new_mask; if (group >= NUM_SCHED_GRPS || group < SCHED_GROUP_NAMED) { - ODP_ERR("Bad group %i\n", group); + _ODP_ERR("Bad group %i\n", group); return -1; } count = odp_thrmask_count(mask); if (count <= 0) { - ODP_ERR("No threads in the mask\n"); + _ODP_ERR("No threads in the mask\n"); return -1; } @@ -1937,7 +1948,7 @@ static int schedule_group_leave(odp_schedule_group_t group, const odp_thrmask_t thr = odp_thrmask_first(mask); for (i = 0; i < count; i++) { if (thr < 0) { - ODP_ERR("No more threads in the mask\n"); + _ODP_ERR("No more threads in the mask\n"); return -1; } @@ -1951,7 +1962,7 @@ static int schedule_group_leave(odp_schedule_group_t group, const odp_thrmask_t if (sched->sched_grp[group].allocated == 0) { odp_ticketlock_unlock(&sched->grp_lock); - ODP_ERR("Bad group status\n"); + _ODP_ERR("Bad group status\n"); return -1; } @@ -2099,14 +2110,14 @@ static void schedule_print(void) (void)schedule_capability(&capa); - ODP_PRINT("\nScheduler debug info\n"); - ODP_PRINT("--------------------\n"); - ODP_PRINT(" scheduler: basic\n"); - ODP_PRINT(" max groups: %u\n", capa.max_groups); - ODP_PRINT(" max priorities: %u\n", capa.max_prios); - ODP_PRINT(" num spread: %i\n", num_spread); - ODP_PRINT(" prefer ratio: %u\n", sched->config.prefer_ratio); - ODP_PRINT("\n"); + _ODP_PRINT("\nScheduler debug info\n"); + _ODP_PRINT("--------------------\n"); + _ODP_PRINT(" scheduler: basic\n"); + _ODP_PRINT(" max groups: %u\n", capa.max_groups); + _ODP_PRINT(" max priorities: %u\n", capa.max_prios); + _ODP_PRINT(" num spread: %i\n", num_spread); + _ODP_PRINT(" prefer ratio: %u\n", sched->config.prefer_ratio); + _ODP_PRINT("\n"); pos = 0; pos += _odp_snprint(&str[pos], size - pos, " Number of active event queues:\n"); @@ -2116,7 +2127,7 @@ static void schedule_print(void) for (spr = 0; spr < num_spread; spr++) pos += _odp_snprint(&str[pos], size - pos, " %7i", spr); - ODP_PRINT("%s\n", str); + _ODP_PRINT("%s\n", str); for (prio = 0; prio < NUM_PRIO; prio++) { for (grp = 0; grp < NUM_SCHED_GRPS; grp++) @@ -2126,7 +2137,7 @@ static void schedule_print(void) if (grp == NUM_SCHED_GRPS) continue; - ODP_PRINT(" prio: %i\n", prio); + _ODP_PRINT(" prio: %i\n", prio); for (grp = 0; grp < NUM_SCHED_GRPS; grp++) { if (sched->sched_grp[grp].allocated == 0) @@ -2143,12 +2154,12 @@ static void schedule_print(void) num_active, num_queues); } - ODP_PRINT("%s\n", str); + _ODP_PRINT("%s\n", str); } } - ODP_PRINT("\n Number of threads per schedule group:\n"); - ODP_PRINT(" name spread\n"); + _ODP_PRINT("\n Number of threads per schedule group:\n"); + _ODP_PRINT(" name spread\n"); for (grp = 0; grp < NUM_SCHED_GRPS; grp++) { if (sched->sched_grp[grp].allocated == 0) @@ -2162,10 +2173,10 @@ static void schedule_print(void) pos += _odp_snprint(&str[pos], size - pos, " %u", sched->sched_grp[grp].spread_thrs[spr]); - ODP_PRINT("%s\n", str); + _ODP_PRINT("%s\n", str); } - ODP_PRINT("\n"); + _ODP_PRINT("\n"); } /* Returns spread for queue debug prints */ @@ -2195,7 +2206,7 @@ const schedule_fn_t _odp_schedule_basic_fn = { }; /* Fill in scheduler API calls */ -const schedule_api_t _odp_schedule_basic_api = { +const _odp_schedule_api_fn_t _odp_schedule_basic_api = { .schedule_wait_time = schedule_wait_time, .schedule_capability = schedule_capability, .schedule_config_init = schedule_config_init, diff --git a/platform/linux-generic/odp_schedule_if.c b/platform/linux-generic/odp_schedule_if.c index 564153137..5f2d6f901 100644 --- a/platform/linux-generic/odp_schedule_if.c +++ b/platform/linux-generic/odp_schedule_if.c @@ -1,5 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,6 +7,8 @@ #include <odp/autoheader_internal.h> +#include <odp/api/plat/schedule_inline_types.h> + #include <odp_schedule_if.h> #include <odp_init_internal.h> #include <odp_debug_internal.h> @@ -15,24 +17,30 @@ #include <stdlib.h> #include <string.h> +/* Enable visibility to inline headers */ +#include <odp/visibility_begin.h> + +const _odp_schedule_api_fn_t *_odp_sched_api; + +int _odp_schedule_configured(void) +{ + return odp_global_rw->schedule_configured; +} + +#include <odp/visibility_end.h> + extern const schedule_fn_t _odp_schedule_sp_fn; -extern const schedule_api_t _odp_schedule_sp_api; +extern const _odp_schedule_api_fn_t _odp_schedule_sp_api; extern const schedule_fn_t _odp_schedule_basic_fn; -extern const schedule_api_t _odp_schedule_basic_api; +extern const _odp_schedule_api_fn_t _odp_schedule_basic_api; extern const schedule_fn_t _odp_schedule_scalable_fn; -extern const schedule_api_t _odp_schedule_scalable_api; +extern const _odp_schedule_api_fn_t _odp_schedule_scalable_api; const schedule_fn_t *_odp_sched_fn; -const schedule_api_t *_odp_sched_api; int _odp_sched_id; -uint64_t odp_schedule_wait_time(uint64_t ns) -{ - return _odp_sched_api->schedule_wait_time(ns); -} - int odp_schedule_capability(odp_schedule_capability_t *capa) { return _odp_sched_api->schedule_capability(capa); @@ -51,7 +59,7 @@ int odp_schedule_config(const odp_schedule_config_t *config) odp_schedule_config_t defconfig; if (odp_global_rw->schedule_configured) { - ODP_ERR("Scheduler has been configured already\n"); + _ODP_ERR("Scheduler has been configured already\n"); return -1; } @@ -68,56 +76,6 @@ int odp_schedule_config(const odp_schedule_config_t *config) return ret; } -odp_event_t odp_schedule(odp_queue_t *from, uint64_t wait) -{ - ODP_ASSERT(odp_global_rw->schedule_configured); - - return _odp_sched_api->schedule(from, wait); -} - -int odp_schedule_multi(odp_queue_t *from, uint64_t wait, odp_event_t events[], - int num) -{ - ODP_ASSERT(odp_global_rw->schedule_configured); - - return _odp_sched_api->schedule_multi(from, wait, events, num); -} - -int odp_schedule_multi_wait(odp_queue_t *from, odp_event_t events[], int num) -{ - return _odp_sched_api->schedule_multi_wait(from, events, num); -} - -int odp_schedule_multi_no_wait(odp_queue_t *from, odp_event_t events[], int num) -{ - return _odp_sched_api->schedule_multi_no_wait(from, events, num); -} - -void odp_schedule_pause(void) -{ - _odp_sched_api->schedule_pause(); -} - -void odp_schedule_resume(void) -{ - _odp_sched_api->schedule_resume(); -} - -void odp_schedule_release_atomic(void) -{ - _odp_sched_api->schedule_release_atomic(); -} - -void odp_schedule_release_ordered(void) -{ - _odp_sched_api->schedule_release_ordered(); -} - -void odp_schedule_prefetch(int num) -{ - _odp_sched_api->schedule_prefetch(num); -} - int odp_schedule_min_prio(void) { return _odp_sched_api->schedule_min_prio(); @@ -178,36 +136,6 @@ int odp_schedule_group_info(odp_schedule_group_t group, return _odp_sched_api->schedule_group_info(group, info); } -void odp_schedule_order_lock(uint32_t lock_index) -{ - _odp_sched_api->schedule_order_lock(lock_index); -} - -void odp_schedule_order_unlock(uint32_t lock_index) -{ - _odp_sched_api->schedule_order_unlock(lock_index); -} - -void odp_schedule_order_unlock_lock(uint32_t unlock_index, uint32_t lock_index) -{ - _odp_sched_api->schedule_order_unlock_lock(unlock_index, lock_index); -} - -void odp_schedule_order_lock_start(uint32_t lock_index) -{ - _odp_sched_api->schedule_order_lock_start(lock_index); -} - -void odp_schedule_order_lock_wait(uint32_t lock_index) -{ - _odp_sched_api->schedule_order_lock_wait(lock_index); -} - -void odp_schedule_order_wait(void) -{ - _odp_sched_api->schedule_order_wait(); -} - void odp_schedule_print(void) { _odp_sched_api->schedule_print(); @@ -220,7 +148,7 @@ int _odp_schedule_init_global(void) if (sched == NULL || !strcmp(sched, "default")) sched = _ODP_SCHEDULE_DEFAULT; - ODP_PRINT("Using scheduler '%s'\n", sched); + _ODP_PRINT("Using scheduler '%s'\n", sched); if (!strcmp(sched, "basic")) { _odp_sched_id = _ODP_SCHED_ID_BASIC; @@ -235,7 +163,7 @@ int _odp_schedule_init_global(void) _odp_sched_fn = &_odp_schedule_scalable_fn; _odp_sched_api = &_odp_schedule_scalable_api; } else { - ODP_ABORT("Unknown scheduler specified via ODP_SCHEDULER\n"); + _ODP_ABORT("Unknown scheduler specified via ODP_SCHEDULER\n"); return -1; } diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 08f19c4e5..f90189d05 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -14,9 +14,11 @@ #include <odp/api/shared_memory.h> #include <odp/api/sync.h> #include <odp/api/thread.h> -#include <odp/api/plat/thread_inlines.h> #include <odp/api/thrmask.h> #include <odp/api/time.h> + +#include <odp/api/plat/schedule_inline_types.h> +#include <odp/api/plat/thread_inlines.h> #include <odp/api/plat/time_inlines.h> #include <odp_config_internal.h> @@ -79,7 +81,7 @@ static int thread_state_init(int tidx) sched_scalable_thread_state_t *ts; uint32_t i; - ODP_ASSERT(tidx < MAXTHREADS); + _ODP_ASSERT(tidx < MAXTHREADS); ts = &global->thread_state[tidx]; ts->atomq = NULL; ts->src_schedq = NULL; @@ -123,7 +125,7 @@ static void insert_schedq_in_list(sched_scalable_thread_state_t *ts, } } if (ts->num_schedq == SCHEDQ_PER_THREAD) - ODP_ABORT("Too many schedqs\n"); + _ODP_ABORT("Too many schedqs\n"); ts->schedq_list[ts->num_schedq++] = schedq; } @@ -139,7 +141,7 @@ static void remove_schedq_from_list(sched_scalable_thread_state_t *ts, ts->num_schedq--; return; } - ODP_ABORT("Cannot find schedq\n"); + _ODP_ABORT("Cannot find schedq\n"); } /******************************************************************************* @@ -319,7 +321,7 @@ sched_update_deq(sched_elem_t *q, */ oss = q->qschst; do { - ODP_ASSERT(oss.cur_ticket == _odp_sched_ts->ticket); + _ODP_ASSERT(oss.cur_ticket == _odp_sched_ts->ticket); nss = oss; nss.numevts -= actual; if (nss.numevts > 0 && !pushed) { @@ -361,7 +363,7 @@ sched_update_deq(sched_elem_t *q, true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); if (odp_unlikely(ticket != TICKET_INVALID)) { - ODP_ASSERT(q->qschst_type != ODP_SCHED_SYNC_ATOMIC); + _ODP_ASSERT(q->qschst_type != ODP_SCHED_SYNC_ATOMIC); /* Wait for our turn to update schedq. */ if (odp_unlikely(__atomic_load_n(&q->qschst.cur_ticket, __ATOMIC_ACQUIRE) != ticket)) { @@ -421,8 +423,8 @@ sched_update_deq_sc(sched_elem_t *q, uint32_t ticket; if (atomic) { - ODP_ASSERT(q->qschst.cur_ticket == _odp_sched_ts->ticket); - ODP_ASSERT(q->qschst.cur_ticket != q->qschst.nxt_ticket); + _ODP_ASSERT(q->qschst.cur_ticket == _odp_sched_ts->ticket); + _ODP_ASSERT(q->qschst.cur_ticket != q->qschst.nxt_ticket); q->qschst.numevts -= actual; q->qschst.cur_ticket = _odp_sched_ts->ticket + 1; if (q->qschst.numevts > 0) @@ -528,9 +530,9 @@ sched_queue_t *_odp_sched_queue_add(odp_schedule_group_t grp, uint32_t prio) sched_group_t *sg; uint32_t x; - ODP_ASSERT(grp >= 0 && grp < (odp_schedule_group_t)MAX_SCHED_GROUP); - ODP_ASSERT((global->sg_free & (1ULL << grp)) == 0); - ODP_ASSERT(prio < ODP_SCHED_PRIO_NUM); + _ODP_ASSERT(grp >= 0 && grp < (odp_schedule_group_t)MAX_SCHED_GROUP); + _ODP_ASSERT((global->sg_free & (1ULL << grp)) == 0); + _ODP_ASSERT(prio < ODP_SCHED_PRIO_NUM); sgi = grp; sg = global->sg_vec[sgi]; @@ -554,9 +556,9 @@ static uint32_t sched_pktin_add(odp_schedule_group_t grp, uint32_t prio) uint32_t sgi; sched_group_t *sg; - ODP_ASSERT(grp >= 0 && grp < (odp_schedule_group_t)MAX_SCHED_GROUP); - ODP_ASSERT((global->sg_free & (1ULL << grp)) == 0); - ODP_ASSERT(prio < ODP_SCHED_PRIO_NUM); + _ODP_ASSERT(grp >= 0 && grp < (odp_schedule_group_t)MAX_SCHED_GROUP); + _ODP_ASSERT((global->sg_free & (1ULL << grp)) == 0); + _ODP_ASSERT(prio < ODP_SCHED_PRIO_NUM); sgi = grp; sg = global->sg_vec[sgi]; @@ -589,9 +591,9 @@ void _odp_sched_queue_rem(odp_schedule_group_t grp, uint32_t prio) sched_group_t *sg; uint32_t x; - ODP_ASSERT(grp >= 0 && grp < (odp_schedule_group_t)MAX_SCHED_GROUP); - ODP_ASSERT((global->sg_free & (1ULL << grp)) == 0); - ODP_ASSERT(prio < ODP_SCHED_PRIO_NUM); + _ODP_ASSERT(grp >= 0 && grp < (odp_schedule_group_t)MAX_SCHED_GROUP); + _ODP_ASSERT((global->sg_free & (1ULL << grp)) == 0); + _ODP_ASSERT(prio < ODP_SCHED_PRIO_NUM); sgi = grp; sg = global->sg_vec[sgi]; @@ -685,8 +687,8 @@ static inline void _schedule_release_atomic(sched_scalable_thread_state_t *ts) { #ifdef CONFIG_QSCHST_LOCK sched_update_deq_sc(ts->atomq, ts->dequeued, true); - ODP_ASSERT(ts->atomq->qschst.cur_ticket != ts->ticket); - ODP_ASSERT(ts->atomq->qschst.cur_ticket == + _ODP_ASSERT(ts->atomq->qschst.cur_ticket != ts->ticket); + _ODP_ASSERT(ts->atomq->qschst.cur_ticket == ts->atomq->qschst.nxt_ticket); #else sched_update_deq(ts->atomq, ts->dequeued, true); @@ -711,10 +713,10 @@ static void pktio_start(int pktio_idx, queue_entry_t *qentry; sched_elem_t *elem; - ODP_ASSERT(pktio_idx < ODP_CONFIG_PKTIO_ENTRIES); + _ODP_ASSERT(pktio_idx < ODP_CONFIG_PKTIO_ENTRIES); for (i = 0; i < num_in_queue; i++) { rxq = in_queue_idx[i]; - ODP_ASSERT(rxq < PKTIO_MAX_QUEUES); + _ODP_ASSERT(rxq < PKTIO_MAX_QUEUES); __atomic_fetch_add(&global->poll_count[pktio_idx], 1, __ATOMIC_RELAXED); qentry = _odp_qentry_from_ext(odpq[i]); @@ -723,7 +725,7 @@ static void pktio_start(int pktio_idx, elem->pktio_idx = pktio_idx; elem->rx_queue = rxq; elem->xoffset = sched_pktin_add(elem->sched_grp, elem->sched_prio); - ODP_ASSERT(elem->schedq != NULL); + _ODP_ASSERT(elem->schedq != NULL); schedq_push(elem->schedq, elem); } } @@ -780,7 +782,7 @@ static int poll_pktin(sched_elem_t *elem, odp_event_t ev[], int num_evts) if (is_ordered(elem)) { /* Need reorder context and slot in reorder window */ rwin = queue_get_rwin((queue_entry_t *)elem); - ODP_ASSERT(rwin != NULL); + _ODP_ASSERT(rwin != NULL); if (odp_unlikely(!have_reorder_ctx(ts) || !_odp_rwin_reserve_sc(rwin, &sn))) { /* Put back queue on source schedq */ @@ -831,7 +833,7 @@ events_dequeued: /* Events remain, enqueue them */ i = _odp_queue_enq_sp(elem, &rx_evts[num], num_rx - num); /* Enqueue must succeed as the queue was empty */ - ODP_ASSERT(i == num_rx - num); + _ODP_ASSERT(i == num_rx - num); } goto events_dequeued; } @@ -858,9 +860,9 @@ events_dequeued: /* Don't push queue to schedq */ } - ODP_ASSERT(ts->atomq == NULL); - ODP_ASSERT(!ts->out_of_order); - ODP_ASSERT(ts->rctx == NULL); + _ODP_ASSERT(ts->atomq == NULL); + _ODP_ASSERT(!ts->out_of_order); + _ODP_ASSERT(ts->rctx == NULL); return 0; } @@ -883,7 +885,7 @@ static int _schedule(odp_queue_t *from, odp_event_t ev[], int num_evts) if (atomq != NULL && is_pktin(atomq)) { /* Atomic pktin queue */ if (ts->dequeued < atomq->qschst.wrr_budget) { - ODP_ASSERT(ts->src_schedq != NULL); + _ODP_ASSERT(ts->src_schedq != NULL); num = poll_pktin(atomq, ev, num_evts); if (odp_likely(num != 0)) { if (from) @@ -896,13 +898,13 @@ static int _schedule(odp_queue_t *from, odp_event_t ev[], int num_evts) } ts->atomq = NULL; } else if (atomq != NULL) { - ODP_ASSERT(ts->ticket != TICKET_INVALID); + _ODP_ASSERT(ts->ticket != TICKET_INVALID); #ifdef CONFIG_QSCHST_LOCK LOCK(&atomq->qschlock); #endif dequeue_atomic: - ODP_ASSERT(ts->ticket == atomq->qschst.cur_ticket); - ODP_ASSERT(ts->ticket != atomq->qschst.nxt_ticket); + _ODP_ASSERT(ts->ticket == atomq->qschst.cur_ticket); + _ODP_ASSERT(ts->ticket != atomq->qschst.nxt_ticket); /* Atomic queues can be dequeued without lock since this thread * has the only reference to the atomic queue being processed. */ @@ -985,7 +987,7 @@ restart_same: #ifdef CONFIG_QSCHST_LOCK LOCK(&atomq->qschlock); ts->ticket = atomq->qschst.nxt_ticket++; - ODP_ASSERT(atomq->qschst.cur_ticket == ts->ticket); + _ODP_ASSERT(atomq->qschst.cur_ticket == ts->ticket); #else /* Dequeued atomic queue from the schedq, only we * can process it and any qschst updates are our @@ -1045,7 +1047,7 @@ restart_same: * Allocate a slot in the reorder window. */ rwin = queue_get_rwin((queue_entry_t *)elem); - ODP_ASSERT(rwin != NULL); + _ODP_ASSERT(rwin != NULL); if (odp_unlikely(!_odp_rwin_reserve(rwin, &sn))) { /* Reorder window full */ /* Look at next schedq, find other queue */ @@ -1107,7 +1109,7 @@ restart_same: * inserted into the reorder window. */ _odp_rctx_release(rctx); - ODP_ASSERT(ts->rctx == NULL); + _ODP_ASSERT(ts->rctx == NULL); } /* Dequeue from parallel/ordered queue failed * Check if we have a queue at the head of the schedq that needs @@ -1137,7 +1139,7 @@ static void schedule_order_lock(uint32_t lock_index) if (odp_unlikely(rctx == NULL || rctx->rwin == NULL || lock_index >= rctx->rwin->lock_count)) { - ODP_ERR("Invalid call to odp_schedule_order_lock\n"); + _ODP_ERR("Invalid call to odp_schedule_order_lock\n"); return; } if (odp_unlikely(__atomic_load_n(&rctx->rwin->olock[lock_index], @@ -1159,7 +1161,7 @@ static void schedule_order_unlock(uint32_t lock_index) rctx->rwin == NULL || lock_index >= rctx->rwin->lock_count || rctx->rwin->olock[lock_index] != rctx->sn)) { - ODP_ERR("Invalid call to odp_schedule_order_unlock\n"); + _ODP_ERR("Invalid call to odp_schedule_order_unlock\n"); return; } atomic_store_release(&rctx->rwin->olock[lock_index], @@ -1456,7 +1458,7 @@ static odp_schedule_group_t schedule_group_create(const char *name, /* Validate inputs */ if (mask == NULL) - ODP_ABORT("mask is NULL\n"); + _ODP_ABORT("mask is NULL\n"); odp_spinlock_lock(&global->sched_grp_lock); @@ -1570,12 +1572,12 @@ static int schedule_group_destroy(odp_schedule_group_t group) /* Check if all threads/queues have left the group */ for (p = 0; p < ODP_SCHED_PRIO_NUM; p++) { if (!bitset_is_null(sg->thr_actual[p])) { - ODP_ERR("Group has threads\n"); + _ODP_ERR("Group has threads\n"); ret = -1; goto thrd_q_present_in_group; } if (p != ODP_SCHED_PRIO_PKTIN && sg->xcount[p] != 0) { - ODP_ERR("Group has queues\n"); + _ODP_ERR("Group has queues\n"); ret = -1; goto thrd_q_present_in_group; } @@ -1606,7 +1608,7 @@ static odp_schedule_group_t schedule_group_lookup(const char *name) /* Validate inputs */ if (name == NULL) - ODP_ABORT("name or mask is NULL\n"); + _ODP_ABORT("name or mask is NULL\n"); group = ODP_SCHED_GROUP_INVALID; @@ -1639,7 +1641,7 @@ static int schedule_group_join(odp_schedule_group_t group, return -1; if (mask == NULL) - ODP_ABORT("name or mask is NULL\n"); + _ODP_ABORT("name or mask is NULL\n"); odp_spinlock_lock(&global->sched_grp_lock); @@ -1671,7 +1673,7 @@ static int schedule_group_leave(odp_schedule_group_t group, } if (mask == NULL) - ODP_ABORT("name or mask is NULL\n"); + _ODP_ABORT("name or mask is NULL\n"); odp_spinlock_lock(&global->sched_grp_lock); @@ -1710,7 +1712,7 @@ static int schedule_group_thrmask(odp_schedule_group_t group, } if (mask == NULL) - ODP_ABORT("name or mask is NULL\n"); + _ODP_ABORT("name or mask is NULL\n"); odp_spinlock_lock(&global->sched_grp_lock); @@ -1748,7 +1750,7 @@ static int schedule_group_info(odp_schedule_group_t group, } if (info == NULL) - ODP_ABORT("name or mask is NULL\n"); + _ODP_ABORT("name or mask is NULL\n"); odp_spinlock_lock(&global->sched_grp_lock); @@ -1794,7 +1796,7 @@ static int schedule_init_global(void) global = odp_shm_addr(shm); if (global == NULL) { - ODP_ERR("Schedule init: Shm reserve failed.\n"); + _ODP_ERR("Schedule init: Shm reserve failed.\n"); return -1; } @@ -1817,7 +1819,7 @@ static int schedule_init_global(void) pool = _odp_ishm_pool_create("sched_shm_pool", pool_size, min_alloc, max_alloc, 0); if (pool == NULL) { - ODP_ERR("Failed to allocate shared memory pool " + _ODP_ERR("Failed to allocate shared memory pool " "for sched\n"); goto failed_sched_shm_pool_create; } @@ -1846,19 +1848,19 @@ static int schedule_init_global(void) odp_thrmask_zero(&mask); tmp_all = odp_schedule_group_create("__group_all", &mask); if (tmp_all != ODP_SCHED_GROUP_ALL) { - ODP_ERR("Could not create ODP_SCHED_GROUP_ALL()\n"); + _ODP_ERR("Could not create ODP_SCHED_GROUP_ALL()\n"); goto failed_create_group_all; } tmp_wrkr = odp_schedule_group_create("__group_worker", &mask); if (tmp_wrkr != ODP_SCHED_GROUP_WORKER) { - ODP_ERR("Could not create ODP_SCHED_GROUP_WORKER()\n"); + _ODP_ERR("Could not create ODP_SCHED_GROUP_WORKER()\n"); goto failed_create_group_worker; } tmp_ctrl = odp_schedule_group_create("__group_control", &mask); if (tmp_ctrl != ODP_SCHED_GROUP_CONTROL) { - ODP_ERR("Could not create ODP_SCHED_GROUP_CONTROL()\n"); + _ODP_ERR("Could not create ODP_SCHED_GROUP_CONTROL()\n"); goto failed_create_group_control; } @@ -1891,21 +1893,21 @@ static int schedule_term_global(void) * GROUP_CONTROL groups. */ if (global->config_if.group_enable.all) { if (odp_schedule_group_destroy(ODP_SCHED_GROUP_ALL) != 0) - ODP_ERR("Failed to destroy ODP_SCHED_GROUP_ALL\n"); + _ODP_ERR("Failed to destroy ODP_SCHED_GROUP_ALL\n"); } if (global->config_if.group_enable.worker) { if (odp_schedule_group_destroy(ODP_SCHED_GROUP_WORKER) != 0) - ODP_ERR("Failed to destroy ODP_SCHED_GROUP_WORKER\n"); + _ODP_ERR("Failed to destroy ODP_SCHED_GROUP_WORKER\n"); } if (global->config_if.group_enable.control) { if (odp_schedule_group_destroy(ODP_SCHED_GROUP_CONTROL) != 0) - ODP_ERR("Failed to destroy ODP_SCHED_GROUP_CONTROL\n"); + _ODP_ERR("Failed to destroy ODP_SCHED_GROUP_CONTROL\n"); } _odp_ishm_pool_destroy(global->sched_shm_pool); if (odp_shm_free(global->shm)) { - ODP_ERR("Shm free failed for scalable scheduler"); + _ODP_ERR("Shm free failed for scalable scheduler"); return -1; } @@ -1931,21 +1933,21 @@ static int schedule_init_local(void) if (global->config_if.group_enable.all) { if (odp_schedule_group_join(ODP_SCHED_GROUP_ALL, &mask) != 0) { - ODP_ERR("Failed to join ODP_SCHED_GROUP_ALL\n"); + _ODP_ERR("Failed to join ODP_SCHED_GROUP_ALL\n"); goto failed_to_join_grp_all; } } if (global->config_if.group_enable.control && thr_type == ODP_THREAD_CONTROL) { if (odp_schedule_group_join(ODP_SCHED_GROUP_CONTROL, &mask) != 0) { - ODP_ERR("Failed to join ODP_SCHED_GROUP_CONTROL\n"); + _ODP_ERR("Failed to join ODP_SCHED_GROUP_CONTROL\n"); goto failed_to_join_grp_ctrl; } } if (global->config_if.group_enable.worker && thr_type == ODP_THREAD_WORKER) { if (odp_schedule_group_join(ODP_SCHED_GROUP_WORKER, &mask) != 0) { - ODP_ERR("Failed to join ODP_SCHED_GROUP_WORKER\n"); + _ODP_ERR("Failed to join ODP_SCHED_GROUP_WORKER\n"); goto failed_to_join_grp_wrkr; } } @@ -1981,25 +1983,24 @@ static int schedule_term_local(void) if (global->config_if.group_enable.all) { if (odp_schedule_group_leave(ODP_SCHED_GROUP_ALL, &mask) != 0) - ODP_ERR("Failed to leave ODP_SCHED_GROUP_ALL\n"); + _ODP_ERR("Failed to leave ODP_SCHED_GROUP_ALL\n"); } if (global->config_if.group_enable.control && thr_type == ODP_THREAD_CONTROL) { if (odp_schedule_group_leave(ODP_SCHED_GROUP_CONTROL, &mask) != 0) - ODP_ERR("Failed to leave ODP_SCHED_GROUP_CONTROL\n"); + _ODP_ERR("Failed to leave ODP_SCHED_GROUP_CONTROL\n"); } if (global->config_if.group_enable.worker && thr_type == ODP_THREAD_WORKER) { if (odp_schedule_group_leave(ODP_SCHED_GROUP_WORKER, &mask) != 0) - ODP_ERR("Failed to leave ODP_SCHED_GROUP_WORKER\n"); + _ODP_ERR("Failed to leave ODP_SCHED_GROUP_WORKER\n"); } update_sg_membership(_odp_sched_ts); /* Check if the thread is still part of any groups */ if (_odp_sched_ts->num_schedq != 0) { - ODP_ERR("Thread %d still part of scheduler group(s)\n", - _odp_sched_ts->tidx); + _ODP_ERR("Thread %d still part of scheduler group(s)\n", _odp_sched_ts->tidx); rc = -1; } @@ -2026,16 +2027,16 @@ static int schedule_config(const odp_schedule_config_t *config) /* Destroy disabled predefined scheduling groups. */ if (!config->sched_group.all) { if (odp_schedule_group_destroy(ODP_SCHED_GROUP_ALL) != 0) - ODP_ERR("Failed to destroy ODP_SCHED_GROUP_ALL\n"); + _ODP_ERR("Failed to destroy ODP_SCHED_GROUP_ALL\n"); } if (!config->sched_group.worker) { if (odp_schedule_group_destroy(ODP_SCHED_GROUP_WORKER) != 0) - ODP_ERR("Failed to destroy ODP_SCHED_GROUP_WORKER\n"); + _ODP_ERR("Failed to destroy ODP_SCHED_GROUP_WORKER\n"); } if (!config->sched_group.control) { if (odp_schedule_group_destroy(ODP_SCHED_GROUP_CONTROL) != 0) - ODP_ERR("Failed to destroy ODP_SCHED_GROUP_CONTROL\n"); + _ODP_ERR("Failed to destroy ODP_SCHED_GROUP_CONTROL\n"); } odp_spinlock_unlock(&global->init_lock); @@ -2131,7 +2132,7 @@ static void order_lock(void) * We are in-order when our reorder window slot number (sn) * equals the head of the reorder window. */ - ODP_ASSERT(ts->rctx != NULL); + _ODP_ASSERT(ts->rctx != NULL); rwin = ts->rctx->rwin; sn = ts->rctx->sn; sevl(); @@ -2178,12 +2179,12 @@ static void schedule_print(void) (void)schedule_capability(&capa); - ODP_PRINT("\nScheduler debug info\n"); - ODP_PRINT("--------------------\n"); - ODP_PRINT(" scheduler: scalable\n"); - ODP_PRINT(" max groups: %u\n", capa.max_groups); - ODP_PRINT(" max priorities: %u\n", capa.max_prios); - ODP_PRINT("\n"); + _ODP_PRINT("\nScheduler debug info\n"); + _ODP_PRINT("--------------------\n"); + _ODP_PRINT(" scheduler: scalable\n"); + _ODP_PRINT(" max groups: %u\n", capa.max_groups); + _ODP_PRINT(" max priorities: %u\n", capa.max_prios); + _ODP_PRINT("\n"); } const schedule_fn_t _odp_schedule_scalable_fn = { @@ -2204,7 +2205,7 @@ const schedule_fn_t _odp_schedule_scalable_fn = { .max_ordered_locks = schedule_max_ordered_locks, }; -const schedule_api_t _odp_schedule_scalable_api = { +const _odp_schedule_api_fn_t _odp_schedule_scalable_api = { .schedule_wait_time = schedule_wait_time, .schedule_capability = schedule_capability, .schedule_config_init = schedule_config_init, diff --git a/platform/linux-generic/odp_schedule_scalable_ordered.c b/platform/linux-generic/odp_schedule_scalable_ordered.c index 54181f10f..f6655d7fa 100644 --- a/platform/linux-generic/odp_schedule_scalable_ordered.c +++ b/platform/linux-generic/odp_schedule_scalable_ordered.c @@ -97,7 +97,7 @@ bool _odp_rwin_reserve_sc(reorder_window_t *rwin, uint32_t *sn) void _odp_rwin_unreserve_sc(reorder_window_t *rwin, uint32_t sn) { - ODP_ASSERT(rwin->tail == sn + 1); + _ODP_ASSERT(rwin->tail == sn + 1); rwin->tail = sn; } @@ -117,7 +117,7 @@ static void rwin_insert(reorder_window_t *rwin, /* We are out-of-order. Store context in reorder window, * releasing its content. */ - ODP_ASSERT(rwin->ring[sn & winmask] == NULL); + _ODP_ASSERT(rwin->ring[sn & winmask] == NULL); atomic_store_release(&rwin->ring[sn & winmask], rctx, /*readonly=*/false); @@ -147,7 +147,7 @@ static void rwin_insert(reorder_window_t *rwin, } /* old.head == sn => we are now in-order! */ - ODP_ASSERT(old.head == sn); + _ODP_ASSERT(old.head == sn); /* We are in-order so our responsibility to retire contexts */ new.head = old.head; new.chgi = old.chgi + 1; @@ -191,7 +191,7 @@ void _odp_rctx_init(reorder_context_t *rctx, uint16_t idx, /* rctx->rvec_free and rctx->idx already initialised in * thread_state_init function. */ - ODP_ASSERT(rctx->idx == idx); + _ODP_ASSERT(rctx->idx == idx); rctx->rwin = rwin; rctx->sn = sn; rctx->olock_flags = 0; @@ -210,7 +210,7 @@ static inline void rctx_free(const reorder_context_t *rctx) next_idx = rctx->next_idx; - ODP_ASSERT(rctx->rwin != NULL); + _ODP_ASSERT(rctx->rwin != NULL); /* Set free bit */ if (rctx->rvec_free == &_odp_sched_ts->rvec_free) /* Since it is our own reorder context, we can instead @@ -266,7 +266,7 @@ static void blocking_enqueue(queue_entry_t *q, _odp_event_hdr_t **evts, int num) /* Attempt to enqueue remaining events */ actual = q->enqueue_multi(qentry_to_int(q), evts, num); if (odp_unlikely(actual < 0)) - ODP_ERR("Failed to enqueue deferred events\n"); + _ODP_ERR("Failed to enqueue deferred events\n"); /* Update for potential partial success */ evts += actual; num -= actual; @@ -327,7 +327,7 @@ int _odp_rctx_save(queue_entry_t *queue, _odp_event_hdr_t *event_hdr[], int num) ts = _odp_sched_ts; first = ts->rctx; - ODP_ASSERT(ts->rctx != NULL); + _ODP_ASSERT(ts->rctx != NULL); cur = &first[(int)first->cur_idx - (int)first->idx]; for (i = 0; i < num; i++) { if (odp_unlikely(cur->numevts == RC_EVT_SIZE)) { diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index 4c4034782..c829f9044 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -22,6 +22,8 @@ #include <odp/api/schedule.h> #include <odp/api/shared_memory.h> +#include <odp/api/plat/schedule_inline_types.h> + #include <odp_schedule_if.h> #include <odp_debug_internal.h> #include <odp_config_internal.h> @@ -171,7 +173,7 @@ static int init_global(void) odp_shm_t shm; sched_group_t *sched_group = NULL; - ODP_DBG("Using SP scheduler\n"); + _ODP_DBG("Using SP scheduler\n"); shm = odp_shm_reserve("_odp_sched_sp_global", sizeof(sched_global_t), @@ -180,7 +182,7 @@ static int init_global(void) sched_global = odp_shm_addr(shm); if (sched_global == NULL) { - ODP_ERR("Schedule init: Shm reserve failed.\n"); + _ODP_ERR("Schedule init: Shm reserve failed.\n"); return -1; } @@ -252,7 +254,7 @@ static int term_global(void) if (sched_global->queue_cmd[qi].init) { while (_odp_sched_queue_deq(qi, &event, 1, 1) > 0) { if (report) { - ODP_ERR("Queue not empty\n"); + _ODP_ERR("Queue not empty\n"); report = 0; } odp_event_free(event); @@ -262,7 +264,7 @@ static int term_global(void) ret = odp_shm_free(sched_global->shm); if (ret < 0) { - ODP_ERR("Shm free failed for sp_scheduler"); + _ODP_ERR("Shm free failed for sp_scheduler"); ret = -1; } @@ -290,7 +292,7 @@ static void schedule_group_clear(odp_schedule_group_t group) const odp_thrmask_t *thrmask; if (group < 0 || group >= NUM_STATIC_GROUP) - ODP_ABORT("Invalid scheduling group\n"); + _ODP_ABORT("Invalid scheduling group\n"); thrmask = &sched_group->s.group[group].mask; @@ -356,7 +358,7 @@ static void remove_group(sched_group_t *sched_group, int thr, int group) /* Extra array bounds check to suppress warning on GCC 7.4 with -O3 */ if (num >= NUM_GROUP) { - ODP_ERR("Too many groups"); + _ODP_ERR("Too many groups"); return; } @@ -440,7 +442,7 @@ static int create_queue(uint32_t qi, const odp_schedule_param_t *sched_param) int prio = 0; if (odp_global_rw->schedule_configured == 0) { - ODP_ERR("Scheduler has not been configured\n"); + _ODP_ERR("Scheduler has not been configured\n"); return -1; } @@ -527,14 +529,12 @@ static void pktio_start(int pktio_index, int i; sched_cmd_t *cmd; - ODP_DBG("pktio index: %i, %i pktin queues %i\n", - pktio_index, num, pktin_idx[0]); + _ODP_DBG("pktio index: %i, %i pktin queues %i\n", pktio_index, num, pktin_idx[0]); cmd = &sched_global->pktio_cmd[pktio_index]; if (num > NUM_PKTIN) - ODP_ABORT("Supports only %i pktin queues per interface\n", - NUM_PKTIN); + _ODP_ABORT("Supports only %i pktin queues per interface\n", NUM_PKTIN); for (i = 0; i < num; i++) { cmd->pktin_idx[i] = pktin_idx[i]; @@ -607,7 +607,7 @@ static inline void enqueue_packets(odp_queue_t queue, if (num_enq < num_pkt) { num_drop = num_pkt - num_enq; - ODP_DBG("Dropped %i packets\n", num_drop); + _ODP_DBG("Dropped %i packets\n", num_drop); odp_packet_free_multi((odp_packet_t *)&hdr_tbl[num_enq], num_drop); } @@ -1046,12 +1046,12 @@ static void schedule_print(void) (void)schedule_capability(&capa); - ODP_PRINT("\nScheduler debug info\n"); - ODP_PRINT("--------------------\n"); - ODP_PRINT(" scheduler: sp\n"); - ODP_PRINT(" max groups: %u\n", capa.max_groups); - ODP_PRINT(" max priorities: %u\n", capa.max_prios); - ODP_PRINT("\n"); + _ODP_PRINT("\nScheduler debug info\n"); + _ODP_PRINT("--------------------\n"); + _ODP_PRINT(" scheduler: sp\n"); + _ODP_PRINT(" max groups: %u\n", capa.max_groups); + _ODP_PRINT(" max priorities: %u\n", capa.max_prios); + _ODP_PRINT("\n"); } static void get_config(schedule_config_t *config) @@ -1080,7 +1080,7 @@ const schedule_fn_t _odp_schedule_sp_fn = { }; /* Fill in scheduler API calls */ -const schedule_api_t _odp_schedule_sp_api = { +const _odp_schedule_api_fn_t _odp_schedule_sp_api = { .schedule_wait_time = schedule_wait_time, .schedule_capability = schedule_capability, .schedule_config_init = schedule_config_init, diff --git a/platform/linux-generic/odp_shared_memory.c b/platform/linux-generic/odp_shared_memory.c index 74e5c9864..4103568a7 100644 --- a/platform/linux-generic/odp_shared_memory.c +++ b/platform/linux-generic/odp_shared_memory.c @@ -72,7 +72,7 @@ odp_shm_t odp_shm_reserve(const char *name, uint64_t size, uint64_t align, uint32_t supported_flgs = SUPPORTED_SHM_FLAGS; if (flags & ~supported_flgs) { - ODP_ERR("Unsupported SHM flag\n"); + _ODP_ERR("Unsupported SHM flag\n"); return ODP_SHM_INVALID; } diff --git a/platform/linux-generic/odp_sorted_list.c b/platform/linux-generic/odp_sorted_list.c index 8f998238d..c93b11886 100644 --- a/platform/linux-generic/odp_sorted_list.c +++ b/platform/linux-generic/odp_sorted_list.c @@ -262,12 +262,12 @@ void _odp_sorted_list_stats_print(_odp_int_sorted_pool_t sorted_pool) sorted_pool_t *pool; pool = (sorted_pool_t *)(uintptr_t)sorted_pool; - ODP_PRINT("sorted_pool=0x%" PRIX64 "\n", sorted_pool); - ODP_PRINT(" max_sorted_lists=%u next_list_idx=%u\n", - pool->max_sorted_lists, pool->next_list_idx); - ODP_PRINT(" total_inserts=%" PRIu64 " total_deletes=%" PRIu64 - " total_removes=%" PRIu64 "\n", pool->total_inserts, - pool->total_deletes, pool->total_removes); + _ODP_PRINT(" sorted_pool=0x%" PRIX64 "\n", sorted_pool); + _ODP_PRINT(" max_sorted_lists=%u next_list_idx=%u\n", + pool->max_sorted_lists, pool->next_list_idx); + _ODP_PRINT(" total_inserts=%" PRIu64 " total_deletes=%" PRIu64 + " total_removes=%" PRIu64 "\n", pool->total_inserts, + pool->total_deletes, pool->total_removes); } void _odp_sorted_pool_destroy(_odp_int_sorted_pool_t sorted_pool) diff --git a/platform/linux-generic/odp_stash.c b/platform/linux-generic/odp_stash.c index 8fe9c1096..c7d4136ab 100644 --- a/platform/linux-generic/odp_stash.c +++ b/platform/linux-generic/odp_stash.c @@ -69,7 +69,7 @@ int _odp_stash_init_global(void) odp_shm_t shm; if (odp_global_ro.disable.stash) { - ODP_PRINT("Stash is DISABLED\n"); + _ODP_PRINT("Stash is DISABLED\n"); return 0; } @@ -79,7 +79,7 @@ int _odp_stash_init_global(void) stash_global = odp_shm_addr(shm); if (stash_global == NULL) { - ODP_ERR("SHM reserve of stash global data failed\n"); + _ODP_ERR("SHM reserve of stash global data failed\n"); return -1; } @@ -99,7 +99,7 @@ int _odp_stash_term_global(void) return 0; if (odp_shm_free(stash_global->shm)) { - ODP_ERR("SHM free failed\n"); + _ODP_ERR("SHM free failed\n"); return -1; } @@ -109,7 +109,7 @@ int _odp_stash_term_global(void) int odp_stash_capability(odp_stash_capability_t *capa, odp_stash_type_t type) { if (odp_global_ro.disable.stash) { - ODP_ERR("Stash is disabled\n"); + _ODP_ERR("Stash is disabled\n"); return -1; } @@ -175,29 +175,29 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) uint32_t shm_flags = 0; if (odp_global_ro.disable.stash) { - ODP_ERR("Stash is disabled\n"); + _ODP_ERR("Stash is disabled\n"); return ODP_STASH_INVALID; } if (param->obj_size > sizeof(uint64_t)) { - ODP_ERR("Too large object handle.\n"); + _ODP_ERR("Too large object handle.\n"); return ODP_STASH_INVALID; } if (param->num_obj > MAX_RING_SIZE) { - ODP_ERR("Too many objects.\n"); + _ODP_ERR("Too many objects.\n"); return ODP_STASH_INVALID; } if (name && strlen(name) >= ODP_STASH_NAME_LEN) { - ODP_ERR("Too long name.\n"); + _ODP_ERR("Too long name.\n"); return ODP_STASH_INVALID; } index = reserve_index(); if (index < 0) { - ODP_ERR("Maximum number of stashes created already.\n"); + _ODP_ERR("Maximum number of stashes created already.\n"); return ODP_STASH_INVALID; } @@ -227,7 +227,7 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) shm = odp_shm_reserve(shm_name, shm_size, ODP_CACHE_LINE_SIZE, shm_flags); if (shm == ODP_SHM_INVALID) { - ODP_ERR("SHM reserve failed.\n"); + _ODP_ERR("SHM reserve failed.\n"); free_index(index); return ODP_STASH_INVALID; } @@ -279,7 +279,7 @@ int odp_stash_destroy(odp_stash_t st) free_index(index); if (odp_shm_free(shm)) { - ODP_ERR("SHM free failed.\n"); + _ODP_ERR("SHM free failed.\n"); return -1; } @@ -390,7 +390,7 @@ static inline int32_t stash_put_u32(odp_stash_t st, const uint32_t val[], if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; - ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); + _ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); ring_u32_enq_multi(&stash->ring_u32.hdr, stash->ring_mask, (uint32_t *)(uintptr_t)val, num); @@ -417,7 +417,7 @@ static inline int32_t stash_put_u64(odp_stash_t st, const uint64_t val[], if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; - ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); + _ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); ring_u64_enq_multi(&stash->ring_u64.hdr, stash->ring_mask, (uint64_t *)(uintptr_t)val, num); @@ -444,7 +444,7 @@ static inline int32_t stash_put_ptr(odp_stash_t st, const uintptr_t ptr[], if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; - ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); + _ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); if (sizeof(uintptr_t) == sizeof(uint32_t)) ring_u32_enq_multi(&stash->ring_u32.hdr, stash->ring_mask, @@ -553,7 +553,7 @@ int32_t odp_stash_get_u32(odp_stash_t st, uint32_t val[], int32_t num) if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; - ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); + _ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); return ring_u32_deq_multi(&stash->ring_u32.hdr, stash->ring_mask, val, num); @@ -566,7 +566,7 @@ int32_t odp_stash_get_u32_batch(odp_stash_t st, uint32_t val[], int32_t num) if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; - ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); + _ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); return ring_u32_deq_batch(&stash->ring_u32.hdr, stash->ring_mask, val, num); } @@ -578,7 +578,7 @@ int32_t odp_stash_get_u64(odp_stash_t st, uint64_t val[], int32_t num) if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; - ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); + _ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); return ring_u64_deq_multi(&stash->ring_u64.hdr, stash->ring_mask, val, num); @@ -591,7 +591,7 @@ int32_t odp_stash_get_u64_batch(odp_stash_t st, uint64_t val[], int32_t num) if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; - ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); + _ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); return ring_u64_deq_batch(&stash->ring_u64.hdr, stash->ring_mask, val, num); } @@ -603,7 +603,7 @@ int32_t odp_stash_get_ptr(odp_stash_t st, uintptr_t ptr[], int32_t num) if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; - ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); + _ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); if (sizeof(uintptr_t) == sizeof(uint32_t)) return ring_u32_deq_multi(&stash->ring_u32.hdr, @@ -623,7 +623,7 @@ int32_t odp_stash_get_ptr_batch(odp_stash_t st, uintptr_t ptr[], int32_t num) if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; - ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); + _ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); if (sizeof(uintptr_t) == sizeof(uint32_t)) return ring_u32_deq_batch(&stash->ring_u32.hdr, stash->ring_mask, @@ -663,19 +663,19 @@ void odp_stash_print(odp_stash_t st) stash_t *stash = (stash_t *)(uintptr_t)st; if (st == ODP_STASH_INVALID) { - ODP_ERR("Bad stash handle\n"); + _ODP_ERR("Bad stash handle\n"); return; } - ODP_PRINT("\nStash info\n"); - ODP_PRINT("----------\n"); - ODP_PRINT(" handle 0x%" PRIx64 "\n", odp_stash_to_u64(st)); - ODP_PRINT(" name %s\n", stash->name); - ODP_PRINT(" index %i\n", stash->index); - ODP_PRINT(" obj size %u\n", stash->obj_size); - ODP_PRINT(" obj count %u\n", stash_obj_count(stash)); - ODP_PRINT(" ring size %u\n", stash->ring_mask + 1); - ODP_PRINT("\n"); + _ODP_PRINT("\nStash info\n"); + _ODP_PRINT("----------\n"); + _ODP_PRINT(" handle 0x%" PRIx64 "\n", odp_stash_to_u64(st)); + _ODP_PRINT(" name %s\n", stash->name); + _ODP_PRINT(" index %i\n", stash->index); + _ODP_PRINT(" obj size %u\n", stash->obj_size); + _ODP_PRINT(" obj count %u\n", stash_obj_count(stash)); + _ODP_PRINT(" ring size %u\n", stash->ring_mask + 1); + _ODP_PRINT("\n"); } int odp_stash_stats(odp_stash_t st, odp_stash_stats_t *stats) @@ -683,7 +683,7 @@ int odp_stash_stats(odp_stash_t st, odp_stash_stats_t *stats) stash_t *stash = (stash_t *)(uintptr_t)st; if (st == ODP_STASH_INVALID) { - ODP_ERR("Bad stash handle\n"); + _ODP_ERR("Bad stash handle\n"); return -1; } diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index ea48ec2e3..4f1d92357 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -55,7 +55,7 @@ static int read_cache_line_size(void) file = fopen(CACHE_LNSZ_FILE, "rt"); if (file == NULL) { /* File not found */ - ODP_PRINT("WARN: unable to read host CPU cache line size. " + _ODP_PRINT("WARN: unable to read host CPU cache line size. " "Using ODP_CACHE_LINE_SIZE instead.\n"); return ODP_CACHE_LINE_SIZE; } @@ -83,13 +83,13 @@ static uint64_t default_huge_page_size(void) while (fgets(str, sizeof(str), file) != NULL) { if (sscanf(str, "Hugepagesize: %8lu kB", &sz) == 1) { - ODP_DBG("defaut hp size is %lu kB\n", sz); + _ODP_DBG("defaut hp size is %lu kB\n", sz); fclose(file); return (uint64_t)sz * 1024; } } - ODP_ERR("unable to get default hp size\n"); + _ODP_ERR("unable to get default hp size\n"); fclose(file); return 0; } @@ -215,7 +215,7 @@ static char *get_hugepage_dir(uint64_t hugepage_sz) while (fgets(buf, sizeof(buf), fd)) { if (strsplit(buf, sizeof(buf), tokens, _FIELDNAME_MAX, split_tok) != _FIELDNAME_MAX) { - ODP_ERR("Error parsing %s\n", proc_mounts); + _ODP_ERR("Error parsing %s\n", proc_mounts); break; /* return NULL */ } @@ -292,14 +292,14 @@ static int system_cache_line(system_info_t *sysinfo) ret = read_cache_line_size(); if (ret == 0) { - ODP_ERR("read_cache_line_size failed.\n"); + _ODP_ERR("read_cache_line_size failed.\n"); return -1; } sysinfo->cache_line_size = ret; if (ret != ODP_CACHE_LINE_SIZE) - ODP_PRINT("WARN: host CPU cache line size and ODP_CACHE_LINE_SIZE don't match.\n"); + _ODP_PRINT("WARN: host CPU cache line size and ODP_CACHE_LINE_SIZE don't match.\n"); return 0; } @@ -324,21 +324,21 @@ static int read_config_file(void) str = "system.cpu_mhz"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } odp_global_ro.system_info.default_cpu_hz = (uint64_t)val * 1000000; str = "system.cpu_mhz_max"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } odp_global_ro.system_info.default_cpu_hz_max = (uint64_t)val * 1000000; str = "system.cpu_hz_static"; if (!_odp_libconfig_lookup_int(str, &val)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } odp_global_ro.system_info.cpu_hz_static = !!val; @@ -348,17 +348,17 @@ static int read_config_file(void) static void print_compiler_info(void) { - ODP_PRINT("Compiler defines:\n"); - ODP_PRINT(" __GCC_ATOMIC_LLONG_LOCK_FREE: %d\n", __GCC_ATOMIC_LLONG_LOCK_FREE); - ODP_PRINT(" __GCC_ATOMIC_LONG_LOCK_FREE: %d\n", __GCC_ATOMIC_LONG_LOCK_FREE); - ODP_PRINT(" __GCC_ATOMIC_INT_LOCK_FREE: %d\n", __GCC_ATOMIC_INT_LOCK_FREE); - ODP_PRINT(" __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16: "); + _ODP_PRINT("Compiler defines:\n"); + _ODP_PRINT(" __GCC_ATOMIC_LLONG_LOCK_FREE: %d\n", __GCC_ATOMIC_LLONG_LOCK_FREE); + _ODP_PRINT(" __GCC_ATOMIC_LONG_LOCK_FREE: %d\n", __GCC_ATOMIC_LONG_LOCK_FREE); + _ODP_PRINT(" __GCC_ATOMIC_INT_LOCK_FREE: %d\n", __GCC_ATOMIC_INT_LOCK_FREE); + _ODP_PRINT(" __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16: "); #ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 - ODP_PRINT("1\n"); + _ODP_PRINT("1\n"); #else - ODP_PRINT("0\n"); + _ODP_PRINT("0\n"); #endif - ODP_PRINT("\n"); + _ODP_PRINT("\n"); } /* @@ -381,7 +381,7 @@ int _odp_system_info_init(void) /* Check that CONFIG_NUM_CPU_IDS is large enough */ num_cpus = get_nprocs_conf(); if (num_cpus > CONFIG_NUM_CPU_IDS) - ODP_ERR("Unable to handle all %d " + _ODP_ERR("Unable to handle all %d " "CPU IDs. Increase CONFIG_NUM_CPU_IDS value.\n", num_cpus); @@ -443,7 +443,7 @@ uint64_t odp_cpu_hz(void) uint64_t odp_cpu_hz_id(int id) { - ODP_ASSERT(id >= 0 && id < CONFIG_NUM_CPU_IDS); + _ODP_ASSERT(id >= 0 && id < CONFIG_NUM_CPU_IDS); if (odp_global_ro.system_info.cpu_hz_static) return cpu_hz_static(id); @@ -490,8 +490,7 @@ int odp_sys_huge_page_size_all(uint64_t size[], int num) /* See: kernel.org: hugetlbpage.txt */ dir = opendir("/sys/kernel/mm/hugepages"); if (!dir) { - ODP_PRINT("Failed to open /sys/kernel/mm/hugepages: %s\n", - strerror(errno)); + _ODP_PRINT("Failed to open /sys/kernel/mm/hugepages: %s\n", strerror(errno)); return 0; } @@ -587,7 +586,7 @@ void odp_sys_info_print(void) num_cpu, cpumask_str); str[len] = '\0'; - ODP_PRINT("%s", str); + _ODP_PRINT("%s", str); _odp_sys_info_print_arch(); } @@ -596,21 +595,21 @@ void odp_sys_config_print(void) { /* Print ODP_CONFIG_FILE default and override values */ if (_odp_libconfig_print()) - ODP_ERR("Config file print failed\n"); + _ODP_ERR("Config file print failed\n"); - ODP_PRINT("\n\nodp_config_internal.h values:\n" + _ODP_PRINT("\n\nodp_config_internal.h values:\n" "-----------------------------\n"); - ODP_PRINT("ODP_CONFIG_POOLS: %i\n", ODP_CONFIG_POOLS); - ODP_PRINT("CONFIG_MAX_PLAIN_QUEUES: %i\n", CONFIG_MAX_PLAIN_QUEUES); - ODP_PRINT("CONFIG_MAX_SCHED_QUEUES: %i\n", CONFIG_MAX_SCHED_QUEUES); - ODP_PRINT("CONFIG_QUEUE_MAX_ORD_LOCKS: %i\n", CONFIG_QUEUE_MAX_ORD_LOCKS); - ODP_PRINT("ODP_CONFIG_PKTIO_ENTRIES: %i\n", ODP_CONFIG_PKTIO_ENTRIES); - ODP_PRINT("CONFIG_PACKET_HEADROOM: %i\n", CONFIG_PACKET_HEADROOM); - ODP_PRINT("CONFIG_PACKET_TAILROOM: %i\n", CONFIG_PACKET_TAILROOM); - ODP_PRINT("CONFIG_SHM_BLOCKS: %i\n", CONFIG_SHM_BLOCKS); - ODP_PRINT("CONFIG_BURST_SIZE: %i\n", CONFIG_BURST_SIZE); - ODP_PRINT("CONFIG_POOL_MAX_NUM: %i\n", CONFIG_POOL_MAX_NUM); - ODP_PRINT("CONFIG_POOL_CACHE_MAX_SIZE: %i\n", CONFIG_POOL_CACHE_MAX_SIZE); - ODP_PRINT("CONFIG_TIMER_128BIT_ATOMICS: %i\n", CONFIG_TIMER_128BIT_ATOMICS); - ODP_PRINT("\n"); + _ODP_PRINT("ODP_CONFIG_POOLS: %i\n", ODP_CONFIG_POOLS); + _ODP_PRINT("CONFIG_MAX_PLAIN_QUEUES: %i\n", CONFIG_MAX_PLAIN_QUEUES); + _ODP_PRINT("CONFIG_MAX_SCHED_QUEUES: %i\n", CONFIG_MAX_SCHED_QUEUES); + _ODP_PRINT("CONFIG_QUEUE_MAX_ORD_LOCKS: %i\n", CONFIG_QUEUE_MAX_ORD_LOCKS); + _ODP_PRINT("ODP_CONFIG_PKTIO_ENTRIES: %i\n", ODP_CONFIG_PKTIO_ENTRIES); + _ODP_PRINT("CONFIG_PACKET_HEADROOM: %i\n", CONFIG_PACKET_HEADROOM); + _ODP_PRINT("CONFIG_PACKET_TAILROOM: %i\n", CONFIG_PACKET_TAILROOM); + _ODP_PRINT("CONFIG_SHM_BLOCKS: %i\n", CONFIG_SHM_BLOCKS); + _ODP_PRINT("CONFIG_BURST_SIZE: %i\n", CONFIG_BURST_SIZE); + _ODP_PRINT("CONFIG_POOL_MAX_NUM: %i\n", CONFIG_POOL_MAX_NUM); + _ODP_PRINT("CONFIG_POOL_CACHE_MAX_SIZE: %i\n", CONFIG_POOL_CACHE_MAX_SIZE); + _ODP_PRINT("CONFIG_TIMER_128BIT_ATOMICS: %i\n", CONFIG_TIMER_128BIT_ATOMICS); + _ODP_PRINT("\n"); } diff --git a/platform/linux-generic/odp_thread.c b/platform/linux-generic/odp_thread.c index ffb116974..bdfd758d7 100644 --- a/platform/linux-generic/odp_thread.c +++ b/platform/linux-generic/odp_thread.c @@ -58,11 +58,11 @@ int _odp_thread_init_global(void) const char *str = "system.thread_count_max"; if (!_odp_libconfig_lookup_int(str, &num_max)) { - ODP_ERR("Config option '%s' not found.\n", str); + _ODP_ERR("Config option '%s' not found.\n", str); return -1; } if (num_max <= 0) { - ODP_ERR("Config option '%s' not valid.\n", str); + _ODP_ERR("Config option '%s' not valid.\n", str); return -1; } if (num_max > ODP_THREAD_COUNT_MAX) @@ -80,8 +80,8 @@ int _odp_thread_init_global(void) memset(thread_globals, 0, sizeof(thread_globals_t)); odp_spinlock_init(&thread_globals->lock); thread_globals->num_max = num_max; - ODP_PRINT("System config:\n"); - ODP_PRINT(" system.thread_count_max: %d\n\n", num_max); + _ODP_PRINT("System config:\n"); + _ODP_PRINT(" system.thread_count_max: %d\n\n", num_max); return 0; } @@ -95,11 +95,11 @@ int _odp_thread_term_global(void) odp_spinlock_unlock(&thread_globals->lock); if (num) - ODP_ERR("%u threads have not called odp_term_local().\n", num); + _ODP_ERR("%u threads have not called odp_term_local().\n", num); ret = odp_shm_free(odp_shm_lookup("_odp_thread_global")); if (ret < 0) - ODP_ERR("shm free failed for _odp_thread_globals"); + _ODP_ERR("shm free failed for _odp_thread_globals"); return ret; } @@ -180,14 +180,14 @@ int _odp_thread_init_local(odp_thread_type_t type) odp_spinlock_unlock(&thread_globals->lock); if (id < 0) { - ODP_ERR("Too many threads\n"); + _ODP_ERR("Too many threads\n"); return -1; } cpu = sched_getcpu(); if (cpu < 0) { - ODP_ERR("getcpu failed\n"); + _ODP_ERR("getcpu failed\n"); return -1; } @@ -245,7 +245,7 @@ int _odp_thread_term_local(void) odp_spinlock_unlock(&thread_globals->lock); if (num < 0) { - ODP_ERR("failed to free thread id %i", id); + _ODP_ERR("failed to free thread id %i", id); return -1; } diff --git a/platform/linux-generic/odp_time.c b/platform/linux-generic/odp_time.c index b591d712a..3498008f1 100644 --- a/platform/linux-generic/odp_time.c +++ b/platform/linux-generic/odp_time.c @@ -57,7 +57,7 @@ odp_time_t _odp_timespec_cur(void) ret = clock_gettime(CLOCK_MONOTONIC_RAW, &sys_time); if (odp_unlikely(ret != 0)) - ODP_ABORT("clock_gettime failed\n"); + _ODP_ABORT("clock_gettime failed\n"); time.nsec = time_spec_diff_nsec(&sys_time, start_time); @@ -73,7 +73,7 @@ static inline uint64_t time_spec_res(void) ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); if (odp_unlikely(ret != 0)) - ODP_ABORT("clock_getres failed\n"); + _ODP_ABORT("clock_getres failed\n"); return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; } @@ -194,8 +194,7 @@ int _odp_time_init_global(void) if (global->hw_freq_hz == 0) return -1; - ODP_PRINT("HW time counter freq: %" PRIu64 " hz\n\n", - global->hw_freq_hz); + _ODP_PRINT("HW time counter freq: %" PRIu64 " hz\n\n", global->hw_freq_hz); global->hw_start = _odp_cpu_global_time(); return 0; diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index c0ca12ae0..90e83a4f2 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -47,6 +47,7 @@ #include <odp_libconfig_internal.h> #include <odp_macros_internal.h> #include <odp_pool_internal.h> +#include <odp_print_internal.h> #include <odp_queue_if.h> #include <odp_timer_internal.h> #include <odp_types_internal.h> @@ -148,12 +149,14 @@ ODP_STATIC_ASSERT(sizeof(tick_buf_t) == 16, "sizeof(tick_buf_t) == 16"); typedef struct { const void *user_ptr; - /* Used for free list when timer is free */ odp_queue_t queue; - /* Period of periodic timer in ticks (nanoseconds), - * includes PERIODIC_CANCELLED flag. */ + + /* Period of periodic timer in ticks (nanoseconds), includes PERIODIC_CANCELLED flag. */ uint64_t periodic_ticks; + /* Used for free list of timers */ + uint32_t next_free; + } _odp_timer_t; typedef struct timer_pool_s { @@ -233,41 +236,11 @@ static timer_global_t *timer_global; /* Timer thread local data */ static __thread timer_local_t timer_local; -/* Forward declarations */ -static void itimer_init(timer_pool_t *tp); -static void itimer_fini(timer_pool_t *tp); - -static void timer_init(_odp_timer_t *tim, tick_buf_t *tb, odp_queue_t _q, const void *_up) -{ - tim->queue = _q; - tim->user_ptr = _up; - tb->tmo_u64 = 0; - tb->tmo_event = ODP_EVENT_INVALID; - - /* Release the timer by setting timer state to inactive */ - odp_atomic_store_rel_u64(&tb->exp_tck, TMO_INACTIVE); -} - -/* Teardown when timer is freed */ -static void timer_fini(_odp_timer_t *tim, tick_buf_t *tb) -{ - ODP_ASSERT(tb->exp_tck.v == TMO_UNUSED); - ODP_ASSERT(tb->tmo_event == ODP_EVENT_INVALID); - tim->queue = ODP_QUEUE_INVALID; - tim->user_ptr = NULL; -} - -static inline uint32_t get_next_free(_odp_timer_t *tim) -{ - /* Reusing 'queue' for next free index */ - return _odp_typeval(tim->queue); -} - static inline void set_next_free(_odp_timer_t *tim, uint32_t nf) { - ODP_ASSERT(tim->queue == ODP_QUEUE_INVALID); - /* Reusing 'queue' for next free index */ - tim->queue = _odp_cast_scalar(odp_queue_t, nf); + _ODP_ASSERT(tim->queue == ODP_QUEUE_INVALID); + + tim->next_free = nf; } static inline timer_pool_t *timer_pool_from_hdl(odp_timer_pool_t hdl) @@ -285,11 +258,11 @@ static inline timer_pool_t *handle_to_tp(odp_timer_t hdl) uint32_t tp_idx = _odp_typeval(hdl) >> INDEX_BITS; timer_pool_t *tp; - ODP_ASSERT(tp_idx < MAX_TIMER_POOLS); + _ODP_ASSERT(tp_idx < MAX_TIMER_POOLS); tp = timer_global->timer_pool[tp_idx]; - ODP_ASSERT(tp != NULL); + _ODP_ASSERT(tp != NULL); return tp; } @@ -299,7 +272,7 @@ static inline uint32_t handle_to_idx(odp_timer_t hdl, { uint32_t idx = (_odp_typeval(hdl) & ((1U << INDEX_BITS) - 1U)) - 1; - ODP_ASSERT(idx < odp_atomic_load_u32(&tp->high_wm)); + _ODP_ASSERT(idx < odp_atomic_load_u32(&tp->high_wm)); __builtin_prefetch(&tp->tick_buf[idx], 0, 0); @@ -309,7 +282,7 @@ static inline uint32_t handle_to_idx(odp_timer_t hdl, static inline odp_timer_t tp_idx_to_handle(timer_pool_t *tp, uint32_t idx) { - ODP_ASSERT((idx + 1) < (1U << INDEX_BITS)); + _ODP_ASSERT((idx + 1) < (1U << INDEX_BITS)); return _odp_cast_scalar(odp_timer_t, (tp->tp_idx << INDEX_BITS) | (idx + 1)); } @@ -338,215 +311,6 @@ static uint64_t max_multiplier_capa(double freq) return mult; } -static odp_timer_pool_t timer_pool_new(const char *name, - const odp_timer_pool_param_t *param) -{ - uint32_t i; - int tp_idx; - size_t sz0, sz1, sz2; - uint64_t tp_size; - uint64_t res_ns, nsec_per_scan; - odp_shm_t shm; - timer_pool_t *tp; - odp_time_t diff, time; - odp_time_t max_diff = ODP_TIME_NULL; - double base_freq = 0.0; - uint64_t max_multiplier = 0; - uint32_t flags = 0; - int periodic = (param->timer_type == ODP_TIMER_TYPE_PERIODIC) ? 1 : 0; - - if (param->res_ns) - res_ns = param->res_ns; - else - res_ns = GIGA_HZ / param->res_hz; - - if (periodic) { - uint64_t max_capa, min_period_ns; - - base_freq = odp_fract_u64_to_dbl(¶m->periodic.base_freq_hz); - max_multiplier = param->periodic.max_multiplier; - - if (base_freq < MIN_BASE_HZ || base_freq > timer_global->max_base_hz) { - ODP_ERR("Bad base frequency: %f\n", base_freq); - return ODP_TIMER_POOL_INVALID; - } - - max_capa = max_multiplier_capa(base_freq); - - if (max_multiplier == 0 || max_multiplier > max_capa) { - ODP_ERR("Bad max multiplier: %" PRIu64 "\n", max_multiplier); - return ODP_TIMER_POOL_INVALID; - } - - min_period_ns = GIGA_HZ / (base_freq * max_multiplier); - - if (res_ns > min_period_ns) - res_ns = min_period_ns; - } - - if (odp_global_ro.shm_single_va) - flags |= ODP_SHM_SINGLE_VA; - - time = odp_time_global(); - - odp_ticketlock_lock(&timer_global->lock); - - if (timer_global->num_timer_pools >= MAX_TIMER_POOLS) { - odp_ticketlock_unlock(&timer_global->lock); - ODP_DBG("No more free timer pools\n"); - return ODP_TIMER_POOL_INVALID; - } - - /* Find timer pool that has not been used for a while, or is used least recently. - * This ensures that inline scan of an old timer pool has completed and its memory - * can be freed. */ - tp_idx = -1; - for (i = 0; i < MAX_TIMER_POOLS; i++) { - if (timer_global->timer_pool_used[i] == 0) { - diff = odp_time_diff(time, timer_global->destroy_time[i]); - - if (odp_time_to_ns(diff) > TIMER_POOL_REUSE_NS) { - tp_idx = i; - break; - } - - if (odp_time_cmp(diff, max_diff) > 0) { - max_diff = diff; - tp_idx = i; - } - } - } - - if (tp_idx < 0) { - odp_ticketlock_unlock(&timer_global->lock); - ODP_DBG("Did not find free timer pool\n"); - return ODP_TIMER_POOL_INVALID; - } - - shm = timer_global->tp_shm[tp_idx]; - timer_global->timer_pool_used[tp_idx] = 1; - timer_global->num_timer_pools++; - - odp_ticketlock_unlock(&timer_global->lock); - - /* Free memory of previously destroyed timer pool */ - if (shm != ODP_SHM_INVALID) { - if (odp_shm_free(shm)) { - ODP_ERR("Failed to free shared memory: tp_idx %i\n", tp_idx); - goto error; - } - } - - sz0 = _ODP_ROUNDUP_CACHE_LINE(sizeof(timer_pool_t)); - sz1 = _ODP_ROUNDUP_CACHE_LINE(sizeof(tick_buf_t) * param->num_timers); - sz2 = _ODP_ROUNDUP_CACHE_LINE(sizeof(_odp_timer_t) * param->num_timers); - tp_size = sz0 + sz1 + sz2; - - shm = odp_shm_reserve(name, tp_size, ODP_CACHE_LINE_SIZE, flags); - - if (odp_unlikely(shm == ODP_SHM_INVALID)) { - ODP_ERR("Timer pool shm reserve failed %" PRIu64 "kB\n", tp_size / 1024); - goto error; - } - - tp = (timer_pool_t *)odp_shm_addr(shm); - memset(tp, 0, tp_size); - - tp->periodic = periodic; - - /* Scan timer pool twice during resolution interval */ - if (res_ns > ODP_TIME_USEC_IN_NS) - nsec_per_scan = res_ns / 2; - else - nsec_per_scan = res_ns; - - tp->nsec_per_scan = nsec_per_scan; - - odp_atomic_init_u64(&tp->cur_tick, 0); - - if (name == NULL) { - tp->name[0] = 0; - } else { - strncpy(tp->name, name, ODP_TIMER_POOL_NAME_LEN - 1); - tp->name[ODP_TIMER_POOL_NAME_LEN - 1] = 0; - } - - tp->param = *param; - tp->param.res_ns = res_ns; - if (periodic) { - tp->base_freq = base_freq; - tp->max_multiplier = max_multiplier; - } else { - tp->min_rel_tck = odp_timer_ns_to_tick(timer_pool_to_hdl(tp), param->min_tmo); - tp->max_rel_tck = odp_timer_ns_to_tick(timer_pool_to_hdl(tp), param->max_tmo); - } - tp->num_alloc = 0; - odp_atomic_init_u32(&tp->high_wm, 0); - tp->first_free = 0; - tp->notify_overrun = 1; - tp->owner = -1; - - if (param->priv) - tp->owner = odp_thread_id(); - - tp->tick_buf = (void *)((char *)odp_shm_addr(shm) + sz0); - tp->timers = (void *)((char *)odp_shm_addr(shm) + sz0 + sz1); - -#if !USE_128BIT_ATOMICS - for (i = 0; i < NUM_LOCKS; i++) - _odp_atomic_flag_clear(&tp->locks[i]); -#endif - - /* Initialize all odp_timer entries */ - for (i = 0; i < tp->param.num_timers; i++) { - tp->timers[i].queue = ODP_QUEUE_INVALID; - set_next_free(&tp->timers[i], i + 1); - tp->timers[i].user_ptr = NULL; - odp_atomic_init_u64(&tp->tick_buf[i].exp_tck, TMO_UNUSED); - tp->tick_buf[i].tmo_event = ODP_EVENT_INVALID; - } - tp->tp_idx = tp_idx; - odp_spinlock_init(&tp->lock); - tp->start_time = odp_time_global(); - - odp_ticketlock_lock(&timer_global->lock); - - /* Inline timer scan may find the timer pool after this */ - odp_mb_release(); - timer_global->timer_pool[tp_idx] = tp; - timer_global->tp_shm[tp_idx] = shm; - - if (timer_global->num_timer_pools == 1) - odp_global_rw->inline_timers = timer_global->use_inline_timers; - - /* Increase poll rate to match the highest resolution */ - if (timer_global->poll_interval_nsec > nsec_per_scan) { - timer_global->poll_interval_nsec = nsec_per_scan; - timer_global->poll_interval_time = - odp_time_global_from_ns(nsec_per_scan); - } - - /* Update the highest index for inline timer scan */ - if (tp_idx > timer_global->highest_tp_idx) - timer_global->highest_tp_idx = tp_idx; - - odp_ticketlock_unlock(&timer_global->lock); - - if (!odp_global_rw->inline_timers) - itimer_init(tp); - - return timer_pool_to_hdl(tp); - -error: - odp_ticketlock_lock(&timer_global->lock); - timer_global->tp_shm[tp_idx] = shm; - timer_global->timer_pool_used[tp_idx] = 0; - timer_global->num_timer_pools--; - odp_ticketlock_unlock(&timer_global->lock); - - return ODP_TIMER_POOL_INVALID; -} - static void block_sigalarm(void) { sigset_t sigset; @@ -556,17 +320,6 @@ static void block_sigalarm(void) sigprocmask(SIG_BLOCK, &sigset, NULL); } -static void stop_timer_thread(timer_pool_t *tp) -{ - int ret; - - ODP_DBG("stop\n"); - tp->thr_exit = 1; - ret = pthread_join(tp->thr_pthread, NULL); - if (ret != 0) - ODP_ABORT("unable to join thread, err %d\n", ret); -} - static void odp_timer_pool_del(timer_pool_t *tp) { int highest; @@ -575,16 +328,25 @@ static void odp_timer_pool_del(timer_pool_t *tp) odp_spinlock_lock(&tp->lock); if (!odp_global_rw->inline_timers) { - /* Stop POSIX itimer signals */ - itimer_fini(tp); - stop_timer_thread(tp); + int ret; + + /* Stop POSIX timer signals */ + if (timer_delete(tp->timerid) != 0) + _ODP_ABORT("timer_delete() returned error %s\n", strerror(errno)); + + /* Stop the thread */ + _ODP_DBG("stop\n"); + tp->thr_exit = 1; + ret = pthread_join(tp->thr_pthread, NULL); + if (ret != 0) + _ODP_ABORT("unable to join thread, err %d\n", ret); } if (tp->num_alloc != 0) { /* It's a programming error to attempt to destroy a */ /* timer pool which is still in use */ odp_spinlock_unlock(&tp->lock); - ODP_ABORT("%s: timers in use\n", tp->name); + _ODP_ABORT("%s: timers in use\n", tp->name); } odp_spinlock_unlock(&tp->lock); @@ -621,13 +383,21 @@ static inline odp_timer_t timer_alloc(timer_pool_t *tp, odp_queue_t queue, const if (odp_likely(tp->num_alloc < tp->param.num_timers)) { tp->num_alloc++; /* Remove first unused timer from free list */ - ODP_ASSERT(tp->first_free != tp->param.num_timers); + _ODP_ASSERT(tp->first_free != tp->param.num_timers); uint32_t idx = tp->first_free; _odp_timer_t *tim = &tp->timers[idx]; + tick_buf_t *tb = &tp->tick_buf[idx]; + + tp->first_free = tim->next_free; + tim->queue = queue; + tim->user_ptr = user_ptr; + tb->tmo_u64 = 0; + tb->tmo_event = ODP_EVENT_INVALID; + + /* Release the timer by setting timer state to inactive */ + odp_atomic_store_rel_u64(&tb->exp_tck, TMO_INACTIVE); + - tp->first_free = get_next_free(tim); - /* Initialize timer */ - timer_init(tim, &tp->tick_buf[idx], queue, user_ptr); if (odp_unlikely(tp->num_alloc > odp_atomic_load_u32(&tp->high_wm))) { /* Update high_wm last with release model to * ensure timer initialization is visible */ @@ -645,38 +415,6 @@ static inline odp_timer_t timer_alloc(timer_pool_t *tp, odp_queue_t queue, const return hdl; } -static odp_event_t timer_set_unused(timer_pool_t *tp, uint32_t idx); - -static inline odp_event_t timer_free(timer_pool_t *tp, uint32_t idx) -{ - _odp_timer_t *tim = &tp->timers[idx]; - - /* Free the timer by setting timer state to unused and - * grab any timeout event */ - odp_event_t old_event = timer_set_unused(tp, idx); - - /* Remove timer from queue */ - _odp_queue_fn->timer_rem(tim->queue); - - /* Destroy timer */ - timer_fini(tim, &tp->tick_buf[idx]); - - /* Insert timer into free list */ - odp_spinlock_lock(&tp->lock); - set_next_free(tim, tp->first_free); - tp->first_free = idx; - ODP_ASSERT(tp->num_alloc != 0); - tp->num_alloc--; - odp_spinlock_unlock(&tp->lock); - - return old_event; -} - -/****************************************************************************** - * Operations on timers - * expire/reset/cancel timer - *****************************************************************************/ - static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, timer_pool_t *tp) { @@ -802,7 +540,7 @@ static odp_event_t timer_set_unused(timer_pool_t *tp, uint32_t idx) _odp_atomic_u128_xchg_mm((_odp_atomic_u128_t *)tb, (_odp_u128_t *)&new, (_odp_u128_t *)&old, - _ODP_MEMMODEL_RLX); + _ODP_MEMMODEL_ACQ_RLS); old_event = old.tmo_event; #else /* Take a related lock */ @@ -825,6 +563,35 @@ static odp_event_t timer_set_unused(timer_pool_t *tp, uint32_t idx) return old_event; } +static inline odp_event_t timer_free(timer_pool_t *tp, uint32_t idx) +{ + _odp_timer_t *tim = &tp->timers[idx]; + tick_buf_t *tb = &tp->tick_buf[idx]; + + /* Free the timer by setting timer state to unused and + * grab any timeout event */ + odp_event_t old_event = timer_set_unused(tp, idx); + + /* Remove timer from queue */ + _odp_queue_fn->timer_rem(tim->queue); + + /* Destroy timer */ + _ODP_ASSERT(tb->exp_tck.v == TMO_UNUSED); + _ODP_ASSERT(tb->tmo_event == ODP_EVENT_INVALID); + tim->queue = ODP_QUEUE_INVALID; + tim->user_ptr = NULL; + + /* Insert timer into free list */ + odp_spinlock_lock(&tp->lock); + set_next_free(tim, tp->first_free); + tp->first_free = idx; + _ODP_ASSERT(tp->num_alloc != 0); + tp->num_alloc--; + odp_spinlock_unlock(&tp->lock); + + return old_event; +} + static odp_event_t timer_cancel(timer_pool_t *tp, uint32_t idx) { tick_buf_t *tb = &tp->tick_buf[idx]; @@ -884,19 +651,25 @@ static odp_event_t timer_cancel(timer_pool_t *tp, uint32_t idx) static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) { + uint64_t exp_tck; + odp_queue_t queue; _odp_timer_t *tim = &tp->timers[idx]; tick_buf_t *tb = &tp->tick_buf[idx]; odp_event_t tmo_event = ODP_EVENT_INVALID; - uint64_t exp_tck; + #if USE_128BIT_ATOMICS /* Atomic re-read for correctness */ - exp_tck = odp_atomic_load_u64(&tb->exp_tck); + exp_tck = odp_atomic_load_acq_u64(&tb->exp_tck); /* Re-check exp_tck */ if (odp_likely(exp_tck <= tick)) { /* Attempt to grab timeout event, replace with inactive timer * and invalid event. */ tick_buf_t new, old; + /* Read queue handle between acq and rel. Timer_free overwrites the handle after + * it sets tick value to inactive. */ + queue = tim->queue; + /* Init all bits, also when tmo_event is less than 64 bits. */ new.tmo_u64 = 0; old.tmo_u64 = 0; @@ -928,6 +701,8 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) if (odp_likely(exp_tck <= tick)) { /* Verify that there is a timeout event */ if (odp_likely(tb->tmo_event != ODP_EVENT_INVALID)) { + queue = tim->queue; + /* Grab timeout event, replace with inactive timer * and invalid event. */ tmo_event = tb->tmo_event; @@ -955,12 +730,11 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) } /* Else ignore events of other types */ /* Post the timeout to the destination queue */ - int rc = odp_queue_enq(tim->queue, tmo_event); + int rc = odp_queue_enq(queue, tmo_event); if (odp_unlikely(rc != 0)) { _odp_event_free(tmo_event); - ODP_ABORT("Failed to enqueue timeout event (%d)\n", - rc); + _ODP_ABORT("Failed to enqueue timeout event (%d)\n", rc); } } } @@ -971,7 +745,7 @@ static inline void timer_pool_scan(timer_pool_t *tp, uint64_t tick) uint32_t high_wm = odp_atomic_load_acq_u32(&tp->high_wm); uint32_t i; - ODP_ASSERT(high_wm <= tp->param.num_timers); + _ODP_ASSERT(high_wm <= tp->param.num_timers); for (i = 0; i < high_wm; i++) { /* As a rare occurrence, we can outsmart the HW prefetcher * and the compiler (GCC -fprefetch-loop-arrays) with some @@ -1043,11 +817,11 @@ static inline void timer_pool_scan_inline(int num, odp_time_t now) if (odp_atomic_cas_u64(&tp->cur_tick, &old_tick, new_tick)) { if (tp->notify_overrun && diff > 1) { if (old_tick == 0) { - ODP_DBG("Timer pool (%s) missed %" PRIi64 " scans in start up\n", - tp->name, diff - 1); + _ODP_DBG("Timer pool (%s) missed %" PRIi64 " scans in start up\n", + tp->name, diff - 1); } else { - ODP_DBG("Timer pool (%s) resolution too high: %" PRIi64 " scans missed\n", - tp->name, diff - 1); + _ODP_DBG("Timer pool (%s) resolution too high: %" PRIi64 " scans missed\n", + tp->name, diff - 1); tp->notify_overrun = 0; } } @@ -1102,8 +876,8 @@ static inline void timer_run_posix(timer_pool_t *tp) if (tp->notify_overrun) { overrun = timer_getoverrun(tp->timerid); if (overrun) { - ODP_DBG("\n\t%d ticks overrun on timer pool \"%s\", timer resolution too high\n", - overrun, tp->name); + _ODP_DBG("\n\t%d ticks overrun on timer pool \"%s\", timer resolution too high\n", + overrun, tp->name); tp->notify_overrun = 0; } } @@ -1189,8 +963,7 @@ static int timer_res_init(void) /* Create timer */ if (timer_create(CLOCK_MONOTONIC, &sigev, &timerid)) - ODP_ABORT("timer_create() returned error %s\n", - strerror(errno)); + _ODP_ABORT("timer_create() returned error %s\n", strerror(errno)); /* Timer resolution start from 1ms */ res = ODP_TIME_MSEC_IN_NS; @@ -1214,8 +987,7 @@ static int timer_res_init(void) ispec.it_value.tv_nsec = (long)nsec; if (timer_settime(timerid, 0, &ispec, NULL)) - ODP_ABORT("timer_settime() returned error %s\n", - strerror(errno)); + _ODP_ABORT("timer_settime() returned error %s\n", strerror(errno)); /* Set signal wait timeout to 10*res */ tmo.tv_sec = 0; tmo.tv_nsec = res * 10; @@ -1236,22 +1008,21 @@ static int timer_res_init(void) timer_res_init_done: timer_global->highest_res_ns *= TIMER_RES_ROUNDUP_FACTOR; if (timer_delete(timerid) != 0) - ODP_ABORT("timer_delete() returned error %s\n", - strerror(errno)); + _ODP_ABORT("timer_delete() returned error %s\n", strerror(errno)); sigemptyset(&sigset); sigprocmask(SIG_BLOCK, &sigset, NULL); return 0; } -static void itimer_init(timer_pool_t *tp) +static void posix_timer_start(timer_pool_t *tp) { struct sigevent sigev; struct itimerspec ispec; uint64_t res, sec, nsec; int ret; - ODP_DBG("Creating POSIX timer for timer pool %s, period %" - PRIu64 " ns\n", tp->name, tp->param.res_ns); + _ODP_DBG("Creating POSIX timer for timer pool %s, period %" PRIu64 " ns\n", + tp->name, tp->param.res_ns); res = tp->param.res_ns; sec = res / ODP_TIME_SEC_IN_NS; @@ -1267,7 +1038,7 @@ static void itimer_init(timer_pool_t *tp) odp_atomic_init_u32(&tp->thr_ready, 0); ret = pthread_create(&tp->thr_pthread, NULL, timer_thread, tp); if (ret) - ODP_ABORT("Unable to create timer thread: %d\n", ret); + _ODP_ABORT("Unable to create timer thread: %d\n", ret); /* wait thread set tp->thr_pid */ while (tp->thr_pid == 0) @@ -1280,8 +1051,7 @@ static void itimer_init(timer_pool_t *tp) sigev.sigev_signo = SIGALRM; if (timer_create(CLOCK_MONOTONIC, &sigev, &tp->timerid)) - ODP_ABORT("timer_create() returned error %s\n", - strerror(errno)); + _ODP_ABORT("timer_create() returned error %s\n", strerror(errno)); memset(&ispec, 0, sizeof(ispec)); ispec.it_interval.tv_sec = (time_t)sec; @@ -1290,8 +1060,7 @@ static void itimer_init(timer_pool_t *tp) ispec.it_value.tv_nsec = (long)nsec; if (timer_settime(tp->timerid, 0, &ispec, NULL)) - ODP_ABORT("timer_settime() returned error %s\n", - strerror(errno)); + _ODP_ABORT("timer_settime() returned error %s\n", strerror(errno)); /* Wait response from timer thread that warm up signals have been * processed. Warm up helps avoiding overrun on the first timeout. */ @@ -1299,11 +1068,212 @@ static void itimer_init(timer_pool_t *tp) sched_yield(); } -static void itimer_fini(timer_pool_t *tp) +static odp_timer_pool_t timer_pool_new(const char *name, const odp_timer_pool_param_t *param) { - if (timer_delete(tp->timerid) != 0) - ODP_ABORT("timer_delete() returned error %s\n", - strerror(errno)); + uint32_t i; + int tp_idx; + size_t sz0, sz1, sz2; + uint64_t tp_size; + uint64_t res_ns, nsec_per_scan; + odp_shm_t shm; + timer_pool_t *tp; + odp_time_t diff, time; + odp_time_t max_diff = ODP_TIME_NULL; + double base_freq = 0.0; + uint64_t max_multiplier = 0; + uint32_t flags = 0; + int periodic = (param->timer_type == ODP_TIMER_TYPE_PERIODIC) ? 1 : 0; + + if (param->res_ns) + res_ns = param->res_ns; + else + res_ns = GIGA_HZ / param->res_hz; + + if (periodic) { + uint64_t max_capa, min_period_ns; + + base_freq = odp_fract_u64_to_dbl(¶m->periodic.base_freq_hz); + max_multiplier = param->periodic.max_multiplier; + + if (base_freq < MIN_BASE_HZ || base_freq > timer_global->max_base_hz) { + _ODP_ERR("Bad base frequency: %f\n", base_freq); + return ODP_TIMER_POOL_INVALID; + } + + max_capa = max_multiplier_capa(base_freq); + + if (max_multiplier == 0 || max_multiplier > max_capa) { + _ODP_ERR("Bad max multiplier: %" PRIu64 "\n", max_multiplier); + return ODP_TIMER_POOL_INVALID; + } + + min_period_ns = GIGA_HZ / (base_freq * max_multiplier); + + if (res_ns > min_period_ns) + res_ns = min_period_ns; + } + + if (odp_global_ro.shm_single_va) + flags |= ODP_SHM_SINGLE_VA; + + time = odp_time_global(); + + odp_ticketlock_lock(&timer_global->lock); + + if (timer_global->num_timer_pools >= MAX_TIMER_POOLS) { + odp_ticketlock_unlock(&timer_global->lock); + _ODP_DBG("No more free timer pools\n"); + return ODP_TIMER_POOL_INVALID; + } + + /* Find timer pool that has not been used for a while, or is used least recently. + * This ensures that inline scan of an old timer pool has completed and its memory + * can be freed. */ + tp_idx = -1; + for (i = 0; i < MAX_TIMER_POOLS; i++) { + if (timer_global->timer_pool_used[i] == 0) { + diff = odp_time_diff(time, timer_global->destroy_time[i]); + + if (odp_time_to_ns(diff) > TIMER_POOL_REUSE_NS) { + tp_idx = i; + break; + } + + if (odp_time_cmp(diff, max_diff) > 0) { + max_diff = diff; + tp_idx = i; + } + } + } + + if (tp_idx < 0) { + odp_ticketlock_unlock(&timer_global->lock); + _ODP_DBG("Did not find free timer pool\n"); + return ODP_TIMER_POOL_INVALID; + } + + shm = timer_global->tp_shm[tp_idx]; + timer_global->timer_pool_used[tp_idx] = 1; + timer_global->num_timer_pools++; + + odp_ticketlock_unlock(&timer_global->lock); + + /* Free memory of previously destroyed timer pool */ + if (shm != ODP_SHM_INVALID) { + if (odp_shm_free(shm)) { + _ODP_ERR("Failed to free shared memory: tp_idx %i\n", tp_idx); + goto error; + } + } + + sz0 = _ODP_ROUNDUP_CACHE_LINE(sizeof(timer_pool_t)); + sz1 = _ODP_ROUNDUP_CACHE_LINE(sizeof(tick_buf_t) * param->num_timers); + sz2 = _ODP_ROUNDUP_CACHE_LINE(sizeof(_odp_timer_t) * param->num_timers); + tp_size = sz0 + sz1 + sz2; + + shm = odp_shm_reserve(name, tp_size, ODP_CACHE_LINE_SIZE, flags); + + if (odp_unlikely(shm == ODP_SHM_INVALID)) { + _ODP_ERR("Timer pool shm reserve failed %" PRIu64 "kB\n", tp_size / 1024); + goto error; + } + + tp = (timer_pool_t *)odp_shm_addr(shm); + memset(tp, 0, tp_size); + + tp->periodic = periodic; + + /* Scan timer pool twice during resolution interval */ + if (res_ns > ODP_TIME_USEC_IN_NS) + nsec_per_scan = res_ns / 2; + else + nsec_per_scan = res_ns; + + tp->nsec_per_scan = nsec_per_scan; + + odp_atomic_init_u64(&tp->cur_tick, 0); + + if (name == NULL) { + tp->name[0] = 0; + } else { + strncpy(tp->name, name, ODP_TIMER_POOL_NAME_LEN - 1); + tp->name[ODP_TIMER_POOL_NAME_LEN - 1] = 0; + } + + tp->param = *param; + tp->param.res_ns = res_ns; + if (periodic) { + tp->base_freq = base_freq; + tp->max_multiplier = max_multiplier; + } else { + tp->min_rel_tck = odp_timer_ns_to_tick(timer_pool_to_hdl(tp), param->min_tmo); + tp->max_rel_tck = odp_timer_ns_to_tick(timer_pool_to_hdl(tp), param->max_tmo); + } + tp->num_alloc = 0; + odp_atomic_init_u32(&tp->high_wm, 0); + tp->first_free = 0; + tp->notify_overrun = 1; + tp->owner = -1; + + if (param->priv) + tp->owner = odp_thread_id(); + + tp->tick_buf = (void *)((char *)odp_shm_addr(shm) + sz0); + tp->timers = (void *)((char *)odp_shm_addr(shm) + sz0 + sz1); + +#if !USE_128BIT_ATOMICS + for (i = 0; i < NUM_LOCKS; i++) + _odp_atomic_flag_clear(&tp->locks[i]); +#endif + + /* Initialize all odp_timer entries */ + for (i = 0; i < tp->param.num_timers; i++) { + tp->timers[i].queue = ODP_QUEUE_INVALID; + set_next_free(&tp->timers[i], i + 1); + tp->timers[i].user_ptr = NULL; + odp_atomic_init_u64(&tp->tick_buf[i].exp_tck, TMO_UNUSED); + tp->tick_buf[i].tmo_event = ODP_EVENT_INVALID; + } + tp->tp_idx = tp_idx; + odp_spinlock_init(&tp->lock); + tp->start_time = odp_time_global(); + + odp_ticketlock_lock(&timer_global->lock); + + /* Inline timer scan may find the timer pool after this */ + odp_mb_release(); + timer_global->timer_pool[tp_idx] = tp; + timer_global->tp_shm[tp_idx] = shm; + + if (timer_global->num_timer_pools == 1) + odp_global_rw->inline_timers = timer_global->use_inline_timers; + + /* Increase poll rate to match the highest resolution */ + if (timer_global->poll_interval_nsec > nsec_per_scan) { + timer_global->poll_interval_nsec = nsec_per_scan; + timer_global->poll_interval_time = + odp_time_global_from_ns(nsec_per_scan); + } + + /* Update the highest index for inline timer scan */ + if (tp_idx > timer_global->highest_tp_idx) + timer_global->highest_tp_idx = tp_idx; + + odp_ticketlock_unlock(&timer_global->lock); + + if (!odp_global_rw->inline_timers) + posix_timer_start(tp); + + return timer_pool_to_hdl(tp); + +error: + odp_ticketlock_lock(&timer_global->lock); + timer_global->tp_shm[tp_idx] = shm; + timer_global->timer_pool_used[tp_idx] = 0; + timer_global->num_timer_pools--; + odp_ticketlock_unlock(&timer_global->lock); + + return ODP_TIMER_POOL_INVALID; } /****************************************************************************** @@ -1315,7 +1285,7 @@ int odp_timer_capability(odp_timer_clk_src_t clk_src, odp_timer_capability_t *capa) { if (clk_src != ODP_CLOCK_DEFAULT) { - ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", clk_src); + _ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", clk_src); return -1; } @@ -1348,12 +1318,12 @@ int odp_timer_res_capability(odp_timer_clk_src_t clk_src, odp_timer_res_capability_t *res_capa) { if (clk_src != ODP_CLOCK_DEFAULT) { - ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", clk_src); + _ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", clk_src); return -1; } if (res_capa->min_tmo) { - ODP_ERR("Only res_ns or max_tmo based quaries supported\n"); + _ODP_ERR("Only res_ns or max_tmo based quaries supported\n"); return -1; } @@ -1376,14 +1346,14 @@ int odp_timer_periodic_capability(odp_timer_clk_src_t clk_src, uint64_t multiplier; if (clk_src != ODP_CLOCK_DEFAULT) { - ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", clk_src); + _ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", clk_src); return -1; } freq = odp_fract_u64_to_dbl(&capa->base_freq_hz); if (freq < MIN_BASE_HZ || freq > timer_global->max_base_hz) { - ODP_ERR("Base frequency not supported (min: %f, max %f)\n", - (double)MIN_BASE_HZ, (double)timer_global->max_base_hz); + _ODP_ERR("Base frequency not supported (min: %f, max %f)\n", + (double)MIN_BASE_HZ, (double)timer_global->max_base_hz); return -1; } @@ -1415,18 +1385,18 @@ odp_timer_pool_t odp_timer_pool_create(const char *name, const odp_timer_pool_param_t *param) { if (odp_global_ro.init_param.not_used.feat.timer) { - ODP_ERR("Trying to use disabled ODP feature.\n"); + _ODP_ERR("Trying to use disabled ODP feature.\n"); return ODP_TIMER_POOL_INVALID; } if (param->clk_src != ODP_CLOCK_DEFAULT) { - ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", param->clk_src); + _ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", param->clk_src); return ODP_TIMER_POOL_INVALID; } if (param->timer_type != ODP_TIMER_TYPE_SINGLE && param->timer_type != ODP_TIMER_TYPE_PERIODIC) { - ODP_ERR("Bad timer type %i\n", param->timer_type); + _ODP_ERR("Bad timer type %i\n", param->timer_type); return ODP_TIMER_POOL_INVALID; } @@ -1473,7 +1443,7 @@ int odp_timer_pool_info(odp_timer_pool_t tpid, odp_timer_pool_info_t *tp_info) timer_pool_t *tp; if (odp_unlikely(tpid == ODP_TIMER_POOL_INVALID)) { - ODP_ERR("Invalid timer pool.\n"); + _ODP_ERR("Invalid timer pool.\n"); return -1; } @@ -1503,12 +1473,12 @@ odp_timer_t odp_timer_alloc(odp_timer_pool_t tpid, odp_queue_t queue, const void timer_pool_t *tp = timer_pool_from_hdl(tpid); if (odp_unlikely(tpid == ODP_TIMER_POOL_INVALID)) { - ODP_ERR("Invalid timer pool.\n"); + _ODP_ERR("Invalid timer pool.\n"); return ODP_TIMER_INVALID; } if (odp_unlikely(queue == ODP_QUEUE_INVALID)) { - ODP_ERR("%s: Invalid queue handle\n", tp->name); + _ODP_ERR("%s: Invalid queue handle\n", tp->name); return ODP_TIMER_INVALID; } /* We don't care about the validity of user_ptr because we will not @@ -1591,7 +1561,7 @@ int odp_timer_start(odp_timer_t timer, const odp_timer_start_t *start_param) /* Check that timer was not active */ if (odp_unlikely(tmo_ev != ODP_EVENT_INVALID)) { - ODP_ERR("Timer was active already\n"); + _ODP_ERR("Timer was active already\n"); odp_event_free(tmo_ev); } @@ -1641,28 +1611,28 @@ int odp_timer_periodic_start(odp_timer_t timer, const odp_timer_periodic_start_t double freq = multiplier * tp->base_freq; if (odp_unlikely(!tp->periodic)) { - ODP_ERR("Not a periodic timer\n"); + _ODP_ERR("Not a periodic timer\n"); return ODP_TIMER_FAIL; } if (odp_unlikely(multiplier == 0 || multiplier > tp->max_multiplier)) { - ODP_ERR("Bad frequency multiplier: %" PRIu64 "\n", multiplier); + _ODP_ERR("Bad frequency multiplier: %" PRIu64 "\n", multiplier); return ODP_TIMER_FAIL; } if (odp_unlikely(odp_event_type(tmo_ev) != ODP_EVENT_TIMEOUT)) { - ODP_ERR("Event type is not timeout\n"); + _ODP_ERR("Event type is not timeout\n"); return ODP_TIMER_FAIL; } period_ns = (uint64_t)((double)ODP_TIME_SEC_IN_NS / freq); if (period_ns == 0) { - ODP_ERR("Too high periodic timer frequency: %f\n", freq); + _ODP_ERR("Too high periodic timer frequency: %f\n", freq); return ODP_TIMER_FAIL; } if (period_ns & PERIODIC_CANCELLED) { - ODP_ERR("Periodic timer frequency error: %f\n", freq); + _ODP_ERR("Periodic timer frequency error: %f\n", freq); return ODP_TIMER_FAIL; } @@ -1684,7 +1654,7 @@ int odp_timer_periodic_start(odp_timer_t timer, const odp_timer_periodic_start_t /* Check that timer was not active */ if (odp_unlikely(tmo_ev != ODP_EVENT_INVALID)) { - ODP_ERR("Timer was active already\n"); + _ODP_ERR("Timer was active already\n"); odp_event_free(tmo_ev); } @@ -1700,7 +1670,7 @@ int odp_timer_periodic_ack(odp_timer_t timer, odp_event_t tmo_ev) _odp_timer_t *tim = &tp->timers[idx]; if (odp_unlikely(odp_event_type(tmo_ev) != ODP_EVENT_TIMEOUT)) { - ODP_ERR("Event type is not timeout\n"); + _ODP_ERR("Event type is not timeout\n"); return -1; } @@ -1720,7 +1690,7 @@ int odp_timer_periodic_ack(odp_timer_t timer, odp_event_t tmo_ev) * timer_reset() call above. */ if (odp_unlikely(tmo_ev != ODP_EVENT_INVALID)) { /* Reset returned an event, free it. */ - ODP_ERR("Timer was active already\n"); + _ODP_ERR("Timer was active already\n"); odp_event_free(tmo_ev); } @@ -1752,14 +1722,14 @@ int odp_timer_periodic_cancel(odp_timer_t hdl) odp_event_t ev; if (odp_unlikely(hdl == ODP_TIMER_INVALID)) { - ODP_ERR("Bad timer pool handle\n"); + _ODP_ERR("Bad timer pool handle\n"); return -1; } tp = handle_to_tp(hdl); if (odp_unlikely(tp->periodic == 0)) { - ODP_ERR("Not a periodic timer\n"); + _ODP_ERR("Not a periodic timer\n"); return -1; } @@ -1775,7 +1745,7 @@ int odp_timer_periodic_cancel(odp_timer_t hdl) /* Timer cancelled and timeout returned. Enqueue tmo, ack call will flag * it as the last event. */ if (odp_unlikely(odp_queue_enq(tim->queue, ev))) { - ODP_ERR("Failed to enqueue timeout event\n"); + _ODP_ERR("Failed to enqueue timeout event\n"); _odp_event_free(ev); return -1; } @@ -1813,11 +1783,11 @@ odp_timeout_t odp_timeout_alloc(odp_pool_t pool_hdl) odp_event_t event; pool_t *pool; - ODP_ASSERT(pool_hdl != ODP_POOL_INVALID); + _ODP_ASSERT(pool_hdl != ODP_POOL_INVALID); pool = _odp_pool_entry(pool_hdl); - ODP_ASSERT(pool->type == ODP_POOL_TIMEOUT); + _ODP_ASSERT(pool->type == ODP_POOL_TIMEOUT); event = _odp_event_alloc(pool); if (odp_unlikely(event == ODP_EVENT_INVALID)) @@ -1834,23 +1804,32 @@ void odp_timeout_free(odp_timeout_t tmo) void odp_timer_pool_print(odp_timer_pool_t timer_pool) { timer_pool_t *tp; + int len = 0; + int max_len = 512; + int n = max_len - 1; + char str[max_len]; if (timer_pool == ODP_TIMER_POOL_INVALID) { - ODP_ERR("Bad timer pool handle\n"); + _ODP_ERR("Bad timer pool handle\n"); return; } tp = timer_pool_from_hdl(timer_pool); - ODP_PRINT("\nTimer pool info\n"); - ODP_PRINT("---------------\n"); - ODP_PRINT(" timer pool %p\n", (void *)tp); - ODP_PRINT(" tp index %u\n", tp->tp_idx); - ODP_PRINT(" num timers %u\n", tp->num_alloc); - ODP_PRINT(" num tp %i\n", timer_global->num_timer_pools); - ODP_PRINT(" inline timers %i\n", timer_global->use_inline_timers); - ODP_PRINT(" periodic %i\n", tp->periodic); - ODP_PRINT("\n"); + len += _odp_snprint(&str[len], n - len, "Timer pool info\n"); + len += _odp_snprint(&str[len], n - len, "---------------\n"); + len += _odp_snprint(&str[len], n - len, " handle 0x%" PRIx64 "\n", + odp_timer_pool_to_u64(timer_pool)); + len += _odp_snprint(&str[len], n - len, " tp index %u\n", tp->tp_idx); + len += _odp_snprint(&str[len], n - len, " num timers %u\n", tp->num_alloc); + len += _odp_snprint(&str[len], n - len, " num tp %i\n", + timer_global->num_timer_pools); + len += _odp_snprint(&str[len], n - len, " inline timers %i\n", + timer_global->use_inline_timers); + len += _odp_snprint(&str[len], n - len, " periodic %i\n", tp->periodic); + str[len] = 0; + + _ODP_PRINT("%s\n", str); } void odp_timer_print(odp_timer_t timer) @@ -1858,9 +1837,13 @@ void odp_timer_print(odp_timer_t timer) timer_pool_t *tp; uint32_t idx; _odp_timer_t *tim; + int len = 0; + int max_len = 512; + int n = max_len - 1; + char str[max_len]; if (timer == ODP_TIMER_INVALID) { - ODP_ERR("Bad timer handle\n"); + _ODP_ERR("Bad timer handle\n"); return; } @@ -1868,45 +1851,63 @@ void odp_timer_print(odp_timer_t timer) idx = handle_to_idx(timer, tp); tim = &tp->timers[idx]; - ODP_PRINT("\nTimer info\n"); - ODP_PRINT("----------\n"); - ODP_PRINT(" timer pool %p\n", (void *)tp); - ODP_PRINT(" timer index %u\n", idx); - ODP_PRINT(" dest queue 0x%" PRIx64 "\n", odp_queue_to_u64(tim->queue)); - ODP_PRINT(" user ptr %p\n", tim->user_ptr); - ODP_PRINT(" periodic ticks %" PRIu64 "\n", tim->periodic_ticks & ~PERIODIC_CANCELLED); - ODP_PRINT("\n"); + len += _odp_snprint(&str[len], n - len, "Timer info\n"); + len += _odp_snprint(&str[len], n - len, "----------\n"); + len += _odp_snprint(&str[len], n - len, " handle 0x%" PRIx64 "\n", + odp_timer_to_u64(timer)); + len += _odp_snprint(&str[len], n - len, " timer pool 0x%" PRIx64 "\n", + odp_timer_pool_to_u64(timer_pool_to_hdl(tp))); + len += _odp_snprint(&str[len], n - len, " timer index %u\n", idx); + len += _odp_snprint(&str[len], n - len, " dest queue 0x%" PRIx64 "\n", + odp_queue_to_u64(tim->queue)); + len += _odp_snprint(&str[len], n - len, " user ptr %p\n", tim->user_ptr); + len += _odp_snprint(&str[len], n - len, " periodic ticks %" PRIu64 "\n", + tim->periodic_ticks & ~PERIODIC_CANCELLED); + str[len] = 0; + + _ODP_PRINT("%s\n", str); } void odp_timeout_print(odp_timeout_t tmo) { const odp_timeout_hdr_t *tmo_hdr; odp_timer_t timer; + int len = 0; + int max_len = 512; + int n = max_len - 1; + char str[max_len]; if (tmo == ODP_TIMEOUT_INVALID) { - ODP_ERR("Bad timeout handle\n"); + _ODP_ERR("Bad timeout handle\n"); return; } tmo_hdr = timeout_hdr(tmo); timer = tmo_hdr->timer; - ODP_PRINT("\nTimeout info\n"); - ODP_PRINT("------------\n"); - ODP_PRINT(" tmo handle 0x%" PRIx64 "\n", odp_timeout_to_u64(tmo)); - ODP_PRINT(" expiration %" PRIu64 "\n", tmo_hdr->expiration); - ODP_PRINT(" user ptr %p\n", tmo_hdr->user_ptr); + len += _odp_snprint(&str[len], n - len, "Timeout info\n"); + len += _odp_snprint(&str[len], n - len, "------------\n"); + len += _odp_snprint(&str[len], n - len, " handle 0x%" PRIx64 "\n", + odp_timeout_to_u64(tmo)); + len += _odp_snprint(&str[len], n - len, " expiration %" PRIu64 "\n", + tmo_hdr->expiration); + len += _odp_snprint(&str[len], n - len, " user ptr %p\n", tmo_hdr->user_ptr); + len += _odp_snprint(&str[len], n - len, " user area %p\n", tmo_hdr->uarea_addr); if (timer != ODP_TIMER_INVALID) { timer_pool_t *tp = handle_to_tp(timer); uint32_t idx = handle_to_idx(timer, tp); - ODP_PRINT(" timer pool %p\n", (void *)tp); - ODP_PRINT(" timer index %u\n", idx); - ODP_PRINT(" periodic %i\n", tp->periodic); + len += _odp_snprint(&str[len], n - len, " timer pool 0x%" PRIx64 "\n", + odp_timer_pool_to_u64(timer_pool_to_hdl(tp))); + len += _odp_snprint(&str[len], n - len, " timer 0x%" PRIx64 "\n", + odp_timer_to_u64(timer)); + len += _odp_snprint(&str[len], n - len, " timer index %u\n", idx); + len += _odp_snprint(&str[len], n - len, " periodic %i\n", tp->periodic); } + str[len] = 0; - ODP_PRINT("\n"); + _ODP_PRINT("%s\n", str); } int _odp_timer_init_global(const odp_init_t *params) @@ -1918,7 +1919,7 @@ int _odp_timer_init_global(const odp_init_t *params) int val = 0; if (params && params->not_used.feat.timer) { - ODP_DBG("Timers disabled\n"); + _ODP_DBG("Timers disabled\n"); timer_global = NULL; return 0; } @@ -1929,7 +1930,7 @@ int _odp_timer_init_global(const odp_init_t *params) timer_global = odp_shm_addr(shm); if (timer_global == NULL) { - ODP_ERR("Shm reserve failed for odp_timer\n"); + _ODP_ERR("Shm reserve failed for odp_timer\n"); return -1; } @@ -1946,47 +1947,47 @@ int _odp_timer_init_global(const odp_init_t *params) } #if USE_128BIT_ATOMICS - ODP_PRINT("Timer using lock-less implementation\n"); + _ODP_PRINT("Timer using lock-less implementation\n"); #else - ODP_PRINT("Timer using lock-based implementation\n"); + _ODP_PRINT("Timer using lock-based implementation\n"); #endif - ODP_PRINT("Timer config:\n"); + _ODP_PRINT("Timer config:\n"); conf_str = "timer.inline"; if (!_odp_libconfig_lookup_int(conf_str, &val)) { - ODP_ERR("Config option '%s' not found.\n", conf_str); + _ODP_ERR("Config option '%s' not found.\n", conf_str); goto error; } timer_global->use_inline_timers = val; - ODP_PRINT(" %s: %i\n", conf_str, val); + _ODP_PRINT(" %s: %i\n", conf_str, val); conf_str = "timer.inline_poll_interval"; if (!_odp_libconfig_lookup_int(conf_str, &val)) { - ODP_ERR("Config option '%s' not found.\n", conf_str); + _ODP_ERR("Config option '%s' not found.\n", conf_str); goto error; } timer_global->poll_interval = val; - ODP_PRINT(" %s: %i\n", conf_str, val); + _ODP_PRINT(" %s: %i\n", conf_str, val); conf_str = "timer.inline_poll_interval_nsec"; if (!_odp_libconfig_lookup_int(conf_str, &val)) { - ODP_ERR("Config option '%s' not found.\n", conf_str); + _ODP_ERR("Config option '%s' not found.\n", conf_str); goto error; } timer_global->poll_interval_nsec = val; timer_global->poll_interval_time = odp_time_global_from_ns(timer_global->poll_interval_nsec); - ODP_PRINT(" %s: %i\n", conf_str, val); + _ODP_PRINT(" %s: %i\n", conf_str, val); conf_str = "timer.inline_thread_type"; if (!_odp_libconfig_lookup_int(conf_str, &val)) { - ODP_ERR("Config option '%s' not found.\n", conf_str); + _ODP_ERR("Config option '%s' not found.\n", conf_str); goto error; } timer_global->thread_type = val; - ODP_PRINT(" %s: %i\n", conf_str, val); - ODP_PRINT("\n"); + _ODP_PRINT(" %s: %i\n", conf_str, val); + _ODP_PRINT("\n"); if (!timer_global->use_inline_timers) { timer_res_init(); @@ -2016,14 +2017,14 @@ int _odp_timer_term_global(void) shm = timer_global->tp_shm[i]; if (shm != ODP_SHM_INVALID) { if (odp_shm_free(shm)) { - ODP_ERR("Shm free failed for timer pool %i\n", i); + _ODP_ERR("Shm free failed for timer pool %i\n", i); return -1; } } } if (odp_shm_free(timer_global->shm)) { - ODP_ERR("Shm free failed for timer_global\n"); + _ODP_ERR("Shm free failed for timer_global\n"); return -1; } diff --git a/platform/linux-generic/odp_timer_wheel.c b/platform/linux-generic/odp_timer_wheel.c index e597d5f70..876e06730 100644 --- a/platform/linux-generic/odp_timer_wheel.c +++ b/platform/linux-generic/odp_timer_wheel.c @@ -940,10 +940,10 @@ uint32_t _odp_timer_wheel_count(_odp_timer_wheel_t timer_wheel) static void _odp_int_timer_wheel_desc_print(wheel_desc_t *wheel_desc, uint32_t wheel_idx) { - ODP_PRINT(" wheel=%u num_slots=%u ticks_shift=%u ticks_per_slot=%u" - " ticks_per_rev=%" PRIu64 "\n", - wheel_idx, wheel_desc->num_slots, wheel_desc->ticks_shift, - wheel_desc->ticks_per_slot, wheel_desc->ticks_per_rev); + _ODP_PRINT(" wheel=%u num_slots=%u ticks_shift=%u ticks_per_slot=%u" + " ticks_per_rev=%" PRIu64 "\n", + wheel_idx, wheel_desc->num_slots, wheel_desc->ticks_shift, + wheel_desc->ticks_per_slot, wheel_desc->ticks_per_rev); } void _odp_timer_wheel_stats_print(_odp_timer_wheel_t timer_wheel) @@ -955,28 +955,28 @@ void _odp_timer_wheel_stats_print(_odp_timer_wheel_t timer_wheel) timer_wheels = (timer_wheels_t *)(uintptr_t)timer_wheel; expired_ring = timer_wheels->expired_timers_ring; - ODP_PRINT("_odp_int_timer_wheel_stats current_ticks=%" PRIu64 "\n", - timer_wheels->current_ticks); + _ODP_PRINT(" _odp_int_timer_wheel_stats current_ticks=%" PRIu64 "\n", + timer_wheels->current_ticks); for (wheel_idx = 0; wheel_idx < 4; wheel_idx++) _odp_int_timer_wheel_desc_print(&timer_wheels->wheel_descs[wheel_idx], wheel_idx); - ODP_PRINT(" total timer_inserts=%" PRIu64 " timer_removes=%" PRIu64 - " insert_fails=%" PRIu64 "\n", - timer_wheels->total_timer_inserts, - timer_wheels->total_timer_removes, - timer_wheels->insert_fail_cnt); - ODP_PRINT(" total_promote_cnt=%" PRIu64 " promote_fail_cnt=%" - PRIu64 "\n", timer_wheels->total_promote_cnt, - timer_wheels->promote_fail_cnt); - ODP_PRINT(" free_list_size=%u min_size=%u peak_size=%u\n", - timer_wheels->free_list_size, - timer_wheels->min_free_list_size, - timer_wheels->peak_free_list_size); - ODP_PRINT(" expired_timers_ring size=%u count=%u " - "peak_count=%u full_cnt=%u\n", - expired_ring->max_idx + 1, expired_ring->count, - expired_ring->peak_count, - expired_ring->expired_ring_full_cnt); + _ODP_PRINT(" total timer_inserts=%" PRIu64 " timer_removes=%" PRIu64 + " insert_fails=%" PRIu64 "\n", + timer_wheels->total_timer_inserts, + timer_wheels->total_timer_removes, + timer_wheels->insert_fail_cnt); + _ODP_PRINT(" total_promote_cnt=%" PRIu64 " promote_fail_cnt=%" + PRIu64 "\n", timer_wheels->total_promote_cnt, + timer_wheels->promote_fail_cnt); + _ODP_PRINT(" free_list_size=%u min_size=%u peak_size=%u\n", + timer_wheels->free_list_size, + timer_wheels->min_free_list_size, + timer_wheels->peak_free_list_size); + _ODP_PRINT(" expired_timers_ring size=%u count=%u " + "peak_count=%u full_cnt=%u\n", + expired_ring->max_idx + 1, expired_ring->count, + expired_ring->peak_count, + expired_ring->expired_ring_full_cnt); } void _odp_timer_wheel_destroy(_odp_timer_wheel_t timer_wheel) diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index 423ef2da4..284a6f7f5 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -366,7 +366,7 @@ static void *alloc_entry_in_tbl(profile_tbl_t *profile_tbl, return profile; } default: - ODP_ERR("Invalid TM profile\n"); + _ODP_ERR("Invalid TM profile\n"); return NULL; } } @@ -401,7 +401,7 @@ static void free_tbl_entry(profile_tbl_t *profile_tbl, return; default: - ODP_ERR("Invalid TM profile\n"); + _ODP_ERR("Invalid TM profile\n"); return; } } @@ -521,7 +521,7 @@ static void *tm_common_profile_create(const char *name, object_ptr = alloc_entry_in_tbl(&tm_glb->profile_tbl, profile_kind, &idx); if (!object_ptr) { - ODP_ERR("No free profiles left\n"); + _ODP_ERR("No free profiles left\n"); return NULL; } @@ -586,7 +586,7 @@ static void *tm_get_profile_params(tm_handle_t profile_handle, return &tm_glb->profile_tbl.wred.profile[idx]; default: - ODP_ERR("Invalid TM profile\n"); + _ODP_ERR("Invalid TM profile\n"); return NULL; } } @@ -769,7 +769,7 @@ static int tm_threshold_config_set(tm_wred_node_t *wred_node, threshold_params = tm_get_profile_params(thresholds_profile, TM_THRESHOLD_PROFILE); if (threshold_params == NULL) { - ODP_DBG("threshold_params is NULL\n"); + _ODP_DBG("threshold_params is NULL\n"); return -1; } @@ -953,7 +953,7 @@ static void tm_block_pkt(tm_system_t *tm_system, tm_queue_obj->timer_shaper, pkt_desc); else if (tm_queue_obj->timer_reason != NO_CALLBACK) - ODP_DBG("%s timer_reason != NO_CALLBACK\n", __func__); + _ODP_DBG("%s timer_reason != NO_CALLBACK\n", __func__); tm_queue_obj->blocked_cnt = 1; tm_queue_obj->blocked_scheduler = schedulers_obj; @@ -983,8 +983,7 @@ static odp_bool_t delay_pkt(tm_system_t *tm_system, rc = _odp_timer_wheel_insert(tm_system->_odp_int_timer_wheel, wakeup_time, timer_context); if (rc < 0) { - ODP_DBG("%s odp_timer_wheel_insert() failed rc=%d\n", - __func__, rc); + _ODP_DBG("%s odp_timer_wheel_insert() failed rc=%d\n", __func__, rc); return false; } @@ -1224,7 +1223,7 @@ static odp_bool_t run_sched(tm_system_t *tm_system, new_sched_state = &schedulers_obj->sched_states[priority]; prev_best_pkt_desc = new_sched_state->smallest_pkt_desc; if (pkt_descs_equal(new_pkt_desc, &prev_best_pkt_desc)) { - ODP_DBG("%s spurious execution ****\n", __func__); + _ODP_DBG("%s spurious execution ****\n", __func__); return false; } } @@ -1675,7 +1674,7 @@ static odp_bool_t tm_consume_pkt_desc(tm_system_t *tm_system, shaper_is_empty = new_shaper_pkt.queue_num == 0; if (pkt_descs_equal(&new_shaper_pkt, sent_pkt_desc)) - ODP_DBG("%s shaper has old pkt_desc\n", __func__); + _ODP_DBG("%s shaper has old pkt_desc\n", __func__); tm_node_obj = shaper_obj->next_tm_node; while (!tm_node_obj->is_root_node) { /* not at egress */ @@ -1685,7 +1684,7 @@ static odp_bool_t tm_consume_pkt_desc(tm_system_t *tm_system, /* Verify that the scheduler output is the sent_pkt_desc. */ if (pkt_descs_not_equal(&prev_sched_pkt, sent_pkt_desc)) { - ODP_DBG("%s sched has bad out pkt_desc\n", __func__); + _ODP_DBG("%s sched has bad out pkt_desc\n", __func__); return false; } @@ -1703,17 +1702,17 @@ static odp_bool_t tm_consume_pkt_desc(tm_system_t *tm_system, sched_is_empty = new_sched_pkt.queue_num == 0; if (pkt_descs_equal(&new_sched_pkt, sent_pkt_desc)) - ODP_DBG("%s sched has old pkt_desc\n", __func__); + _ODP_DBG("%s sched has old pkt_desc\n", __func__); if (pkt_descs_equal(&new_sched_pkt, sent_pkt_desc)) - ODP_DBG("%s scheduler has old pkt_desc\n", __func__); + _ODP_DBG("%s scheduler has old pkt_desc\n", __func__); shaper_obj = &tm_node_obj->shaper_obj; prev_shaper_pkt = shaper_obj->out_pkt_desc; /* Verify that the shaper output is the sent_pkt_desc. */ if (pkt_descs_not_equal(&prev_shaper_pkt, sent_pkt_desc)) { - ODP_DBG("%s shaper has bad out pkt_desc\n", __func__); + _ODP_DBG("%s shaper has bad out pkt_desc\n", __func__); return false; } @@ -1730,7 +1729,7 @@ static odp_bool_t tm_consume_pkt_desc(tm_system_t *tm_system, shaper_is_empty = new_shaper_pkt.queue_num == 0; if (pkt_descs_equal(&new_shaper_pkt, sent_pkt_desc)) - ODP_DBG("%s shaper has old pkt_desc\n", __func__); + _ODP_DBG("%s shaper has old pkt_desc\n", __func__); tm_node_obj = shaper_obj->next_tm_node; } @@ -2055,7 +2054,7 @@ static int tm_enqueue(tm_system_t *tm_system, _odp_sched_fn->order_unlock(); if (rc < 0) { - ODP_DBG("%s work queue full\n", __func__); + _ODP_DBG("%s work queue full\n", __func__); return rc; } @@ -2227,7 +2226,7 @@ static void tm_egress_marking(tm_system_t *tm_system, odp_packet_t odp_pkt) tm_tos_marking_t *ip_marking; color = odp_packet_color(odp_pkt); - ODP_ASSERT(color < ODP_NUM_PACKET_COLORS); + _ODP_ASSERT(color < ODP_NUM_PACKET_COLORS); if (odp_packet_has_vlan(odp_pkt)) { vlan_marking = &tm_system->marking.vlan_marking[color]; @@ -2328,8 +2327,7 @@ static int tm_process_input_work_queue(tm_system_t *tm_system, for (cnt = 1; cnt <= pkts_to_process; cnt++) { rc = input_work_queue_remove(input_work_queue, &work_item); if (rc < 0) { - ODP_DBG("%s input_work_queue_remove() failed\n", - __func__); + _ODP_DBG("%s input_work_queue_remove() failed\n", __func__); return rc; } @@ -2398,7 +2396,7 @@ static int tm_process_expired_timers(tm_system_t *tm_system, if (tm_queue_obj->timer_cancels_outstanding != 0) tm_queue_obj->timer_cancels_outstanding--; else - ODP_DBG("%s bad timer return\n", __func__); + _ODP_DBG("%s bad timer return\n", __func__); return work_done; } @@ -2475,8 +2473,7 @@ static int thread_affinity_get(odp_cpumask_t *odp_cpu_mask) CPU_ZERO(&linux_cpu_set); rc = sched_getaffinity(0, sizeof(cpu_set_t), &linux_cpu_set); if (rc != 0) { - ODP_DBG("%s sched_getaffinity failed with rc=%d\n", - __func__, rc); + _ODP_DBG("%s sched_getaffinity failed with rc=%d\n", __func__, rc); return -1; } @@ -2500,7 +2497,7 @@ static void *tm_system_thread(void *arg) rc = odp_init_local((odp_instance_t)odp_global_ro.main_pid, ODP_THREAD_WORKER); - ODP_ASSERT(rc == 0); + _ODP_ASSERT(rc == 0); tm_group = arg; tm_system = tm_group->first_tm_system; @@ -2564,7 +2561,7 @@ static void *tm_system_thread(void *arg) odp_barrier_wait(&tm_system->tm_system_destroy_barrier); if (odp_term_local() < 0) - ODP_ERR("Term local failed\n"); + _ODP_ERR("Term local failed\n"); return NULL; } @@ -2684,8 +2681,8 @@ int odp_tm_egress_capabilities(odp_tm_capabilities_t *capabilities, if (egress->egress_kind == ODP_TM_EGRESS_PKT_IO) { entry = get_pktio_entry(egress->pktio); if (entry == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", - (uintptr_t)egress->pktio); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", + (uintptr_t)egress->pktio); return -1; } @@ -2827,8 +2824,7 @@ static int affinitize_main_thread(void) if (rc == 0) tm_glb->main_thread_cpu = cpu_num; else - ODP_DBG("%s sched_setaffinity failed with rc=%d\n", - __func__, rc); + _ODP_DBG("%s sched_setaffinity failed with rc=%d\n", __func__, rc); return rc; } @@ -2883,8 +2879,7 @@ static int tm_thread_create(tm_system_group_t *tm_group) rc = pthread_create(&tm_group->thread, &tm_group->attr, tm_system_thread, tm_group); if (rc != 0) - ODP_ERR("Failed to start TM thread on CPU #%u: %d\n", cpu_num, - rc); + _ODP_ERR("Failed to start TM thread on CPU #%u: %d\n", cpu_num, rc); return rc; } @@ -2896,9 +2891,9 @@ static void _odp_tm_group_destroy(_odp_tm_group_t odp_tm_group) tm_group = GET_TM_GROUP(odp_tm_group); /* Wait for the thread to exit. */ - ODP_ASSERT(tm_group->num_tm_systems <= 1); + _ODP_ASSERT(tm_group->num_tm_systems <= 1); rc = pthread_join(tm_group->thread, NULL); - ODP_ASSERT(rc == 0); + _ODP_ASSERT(rc == 0); pthread_attr_destroy(&tm_group->attr); if (tm_glb->cpu_num > 0) tm_glb->cpu_num--; @@ -3059,18 +3054,18 @@ odp_tm_t odp_tm_create(const char *name, int rc; if (odp_global_ro.disable.traffic_mngr) { - ODP_ERR("TM has been disabled\n"); + _ODP_ERR("TM has been disabled\n"); return ODP_TM_INVALID; } if (odp_global_ro.init_param.mem_model == ODP_MEM_MODEL_PROCESS) { - ODP_ERR("TM is not supported in process mode\n"); + _ODP_ERR("TM is not supported in process mode\n"); return ODP_TM_INVALID; } /* We only support global pkt priority mode */ if (requirements->pkt_prio_mode != ODP_TM_PKT_PRIO_MODE_PRESERVE) { - ODP_ERR("Unsupported Packet priority mode\n"); + _ODP_ERR("Unsupported Packet priority mode\n"); return ODP_TM_INVALID; } odp_ticketlock_lock(&tm_glb->create_lock); @@ -4617,7 +4612,7 @@ int odp_tm_enq_multi_lso(odp_tm_queue_t tm_queue, const odp_packet_t packets[], const odp_packet_lso_opt_t *opt_ptr = &lso_opt; if (odp_unlikely(num <= 0)) { - ODP_ERR("No packets\n"); + _ODP_ERR("No packets\n"); return -1; } @@ -4632,7 +4627,7 @@ int odp_tm_enq_multi_lso(odp_tm_queue_t tm_queue, const odp_packet_t packets[], odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); if (pkt_hdr->p.flags.lso == 0) { - ODP_ERR("No LSO options on packet %i\n", i); + _ODP_ERR("No LSO options on packet %i\n", i); goto error; } /* Fill in LSO options from packet */ @@ -4644,14 +4639,14 @@ int odp_tm_enq_multi_lso(odp_tm_queue_t tm_queue, const odp_packet_t packets[], /* Calculate number of packets */ num_pkt = _odp_lso_num_packets(pkt, opt_ptr, &payload_len, &left_over_len); if (odp_unlikely(num_pkt <= 0)) { - ODP_DBG("LSO num packets failed on packet %i\n", i); + _ODP_DBG("LSO num packets failed on packet %i\n", i); goto error; } if (odp_unlikely(num_pkt == 1)) { /* Segmentation not needed */ if (odp_tm_enq_multi(tm_queue, &pkt, 1) != 1) { - ODP_DBG("TM enqueue failed on packet %i\n", i); + _ODP_DBG("TM enqueue failed on packet %i\n", i); goto error; } @@ -4672,7 +4667,7 @@ int odp_tm_enq_multi_lso(odp_tm_queue_t tm_queue, const odp_packet_t packets[], ret = odp_tm_enq_multi(tm_queue, pkt_out, num_pkt); if (odp_unlikely(ret < num_pkt)) { - ODP_DBG("TM enqueue failed on packet %i\n", i); + _ODP_DBG("TM enqueue failed on packet %i\n", i); if (ret < 0) ret = 0; @@ -4978,19 +4973,20 @@ void odp_tm_stats_print(odp_tm_t odp_tm) tm_system = GET_TM_SYSTEM(odp_tm); input_work_queue = &tm_system->input_work_queue; - ODP_PRINT("odp_tm_stats_print - tm_system=0x%" PRIX64 " tm_idx=%u\n", - odp_tm, tm_system->tm_idx); - ODP_PRINT(" input_work_queue size=%u current cnt=%" PRIu64 " peak cnt=%" PRIu32 "\n", - INPUT_WORK_RING_SIZE, odp_atomic_load_u64(&input_work_queue->queue_cnt), - input_work_queue->peak_cnt); - ODP_PRINT(" input_work_queue enqueues=%" PRIu64 " dequeues=%" PRIu64 - " fail_cnt=%" PRIu64 "\n", input_work_queue->total_enqueues, - input_work_queue->total_dequeues, - input_work_queue->enqueue_fail_cnt); - ODP_PRINT(" green_cnt=%" PRIu64 " yellow_cnt=%" PRIu64 " red_cnt=%" - PRIu64 "\n", tm_system->shaper_green_cnt, - tm_system->shaper_yellow_cnt, - tm_system->shaper_red_cnt); + _ODP_PRINT("\nTM stats\n"); + _ODP_PRINT("--------\n"); + _ODP_PRINT(" tm_system=0x%" PRIX64 " tm_idx=%u\n", odp_tm, tm_system->tm_idx); + _ODP_PRINT(" input_work_queue size=%u current cnt=%" PRIu64 " peak cnt=%" PRIu32 "\n", + INPUT_WORK_RING_SIZE, odp_atomic_load_u64(&input_work_queue->queue_cnt), + input_work_queue->peak_cnt); + _ODP_PRINT(" input_work_queue enqueues=%" PRIu64 " dequeues=%" PRIu64 + " fail_cnt=%" PRIu64 "\n", input_work_queue->total_enqueues, + input_work_queue->total_dequeues, + input_work_queue->enqueue_fail_cnt); + _ODP_PRINT(" green_cnt=%" PRIu64 " yellow_cnt=%" PRIu64 " red_cnt=%" PRIu64 "\n", + tm_system->shaper_green_cnt, + tm_system->shaper_yellow_cnt, + tm_system->shaper_red_cnt); _odp_pkt_queue_stats_print(tm_system->_odp_int_queue_pool); _odp_timer_wheel_stats_print(tm_system->_odp_int_timer_wheel); @@ -4999,14 +4995,14 @@ void odp_tm_stats_print(odp_tm_t odp_tm) for (queue_num = 1; queue_num <= ODP_TM_MAX_TM_QUEUES; queue_num++) { tm_queue_obj = tm_system->queue_num_tbl[queue_num - 1]; if (tm_queue_obj && tm_queue_obj->pkts_rcvd_cnt != 0) - ODP_PRINT("queue_num=%u priority=%u rcvd=%u enqueued=%u " - "dequeued=%u consumed=%u\n", - queue_num, - tm_queue_obj->priority, - tm_queue_obj->pkts_rcvd_cnt, - tm_queue_obj->pkts_enqueued_cnt, - tm_queue_obj->pkts_dequeued_cnt, - tm_queue_obj->pkts_consumed_cnt); + _ODP_PRINT("queue_num=%u priority=%u rcvd=%u enqueued=%u " + "dequeued=%u consumed=%u\n", + queue_num, + tm_queue_obj->priority, + tm_queue_obj->pkts_rcvd_cnt, + tm_queue_obj->pkts_enqueued_cnt, + tm_queue_obj->pkts_dequeued_cnt, + tm_queue_obj->pkts_consumed_cnt); } } @@ -5015,7 +5011,7 @@ int odp_tm_queue_stats(odp_tm_queue_t tm_queue, odp_tm_queue_stats_t *stats) tm_queue_obj_t *tm_queue_obj = GET_TM_QUEUE_OBJ(tm_queue); if (!tm_queue_obj) { - ODP_ERR("Invalid TM queue handle\n"); + _ODP_ERR("Invalid TM queue handle\n"); return -1; } @@ -5067,7 +5063,7 @@ int _odp_tm_init_global(void) odp_shm_t shm; if (odp_global_ro.disable.traffic_mngr) { - ODP_PRINT("\nODP traffic manager is DISABLED\n"); + _ODP_PRINT("\nODP traffic manager is DISABLED\n"); return 0; } @@ -5104,7 +5100,7 @@ int _odp_tm_term_global(void) return 0; if (odp_shm_free(tm_glb->shm)) { - ODP_ERR("shm free failed\n"); + _ODP_ERR("shm free failed\n"); return -1; } return 0; diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index fdb448e9c..30e6a0e61 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -58,7 +58,6 @@ #endif #include <ctype.h> -#include <math.h> #include <sched.h> #include <stdint.h> #include <unistd.h> @@ -216,8 +215,7 @@ static int lookup_opt(const char *opt_name, const char *drv_name, int *val) ret = _odp_libconfig_lookup_ext_int(base, drv_name, opt_name, val); if (ret == 0) - ODP_ERR("Unable to find DPDK configuration option: %s\n", - opt_name); + _ODP_ERR("Unable to find DPDK configuration option: %s\n", opt_name); return ret; } @@ -234,7 +232,7 @@ static int init_options(pktio_entry_t *pktio_entry, if (opt->num_rx_desc < dev_info->rx_desc_lim.nb_min || opt->num_rx_desc > dev_info->rx_desc_lim.nb_max || opt->num_rx_desc % dev_info->rx_desc_lim.nb_align) { - ODP_ERR("Invalid number of RX descriptors\n"); + _ODP_ERR("Invalid number of RX descriptors\n"); return -1; } @@ -254,12 +252,12 @@ static int init_options(pktio_entry_t *pktio_entry, return -1; opt->multicast_en = !!val; - ODP_DBG("DPDK interface (%s): %" PRIu16 "\n", dev_info->driver_name, - pkt_priv(pktio_entry)->port_id); - ODP_DBG(" multicast_en: %d\n", opt->multicast_en); - ODP_DBG(" num_rx_desc: %d\n", opt->num_rx_desc); - ODP_DBG(" num_tx_desc: %d\n", opt->num_tx_desc_default); - ODP_DBG(" rx_drop_en: %d\n", opt->rx_drop_en); + _ODP_DBG("DPDK interface (%s): %" PRIu16 "\n", dev_info->driver_name, + pkt_priv(pktio_entry)->port_id); + _ODP_DBG(" multicast_en: %d\n", opt->multicast_en); + _ODP_DBG(" num_rx_desc: %d\n", opt->num_rx_desc); + _ODP_DBG(" num_tx_desc: %d\n", opt->num_tx_desc_default); + _ODP_DBG(" rx_drop_en: %d\n", opt->rx_drop_en); return 0; } @@ -267,15 +265,15 @@ static int init_options(pktio_entry_t *pktio_entry, /** * Calculate valid cache size for DPDK packet pool */ -static unsigned cache_size(uint32_t num) +static uint32_t cache_size(uint32_t num) { - unsigned size = 0; - unsigned i; + uint32_t size = 0; + uint32_t i; if (!RTE_MEMPOOL_CACHE_MAX_SIZE) return 0; - i = ceil((double)num / RTE_MEMPOOL_CACHE_MAX_SIZE); + i = (num + RTE_MEMPOOL_CACHE_MAX_SIZE - 1) / RTE_MEMPOOL_CACHE_MAX_SIZE; i = RTE_MAX(i, 2UL); for (; i <= (num / 2); ++i) if ((num % i) == 0) { @@ -284,7 +282,7 @@ static unsigned cache_size(uint32_t num) } if (odp_unlikely(size > RTE_MEMPOOL_CACHE_MAX_SIZE || (uint32_t)size * 1.5 > num)) { - ODP_ERR("Cache size calc failure: %d\n", size); + _ODP_ERR("Cache size calc failure: %d\n", size); size = 0; } @@ -340,7 +338,7 @@ static void pktmbuf_init(struct rte_mempool *mp, void *opaque_arg ODP_UNUSED, m->buf_iova = rte_mem_virt2iova(buf_addr); if (odp_unlikely(m->buf_iova == 0)) - ODP_ABORT("Bad IO virtual address\n"); + _ODP_ABORT("Bad IO virtual address\n"); m->buf_len = (uint16_t)buf_len; m->data_off = RTE_PKTMBUF_HEADROOM; @@ -372,28 +370,28 @@ static struct rte_mempool *mbuf_pool_create(const char *name, int ret; if (!(pool_entry->mem_from_huge_pages)) { - ODP_ERR("DPDK requires memory is allocated from huge pages\n"); + _ODP_ERR("DPDK requires memory is allocated from huge pages\n"); goto fail; } if (pool_entry->seg_len < RTE_MBUF_DEFAULT_BUF_SIZE) { - ODP_ERR("Some NICs need at least %dB buffers to not segment " - "standard ethernet frames. Increase pool seg_len.\n", - RTE_MBUF_DEFAULT_BUF_SIZE); + _ODP_ERR("Some NICs need at least %dB buffers to not segment " + "standard ethernet frames. Increase pool seg_len.\n", + RTE_MBUF_DEFAULT_BUF_SIZE); goto fail; } if (odp_shm_info(pool_entry->shm, &shm_info)) { - ODP_ERR("Failed to query SHM info.\n"); + _ODP_ERR("Failed to query SHM info.\n"); goto fail; } page_size = shm_info.page_size; total_size = rte_mempool_calc_obj_size(elt_size, MEMPOOL_FLAGS, &sz); if (total_size != pool_entry->block_size) { - ODP_ERR("DPDK pool block size not matching to ODP pool: " - "%" PRIu32 "/%" PRIu32 "\n", total_size, - pool_entry->block_size); + _ODP_ERR("DPDK pool block size not matching to ODP pool: " + "%" PRIu32 "/%" PRIu32 "\n", total_size, + pool_entry->block_size); goto fail; } @@ -401,14 +399,14 @@ static struct rte_mempool *mbuf_pool_create(const char *name, sizeof(struct rte_pktmbuf_pool_private), rte_socket_id(), MEMPOOL_FLAGS); if (mp == NULL) { - ODP_ERR("Failed to create empty DPDK packet pool\n"); + _ODP_ERR("Failed to create empty DPDK packet pool\n"); goto fail; } mp->pool_data = _odp_pool_handle(pool_entry); if (rte_mempool_set_ops_byname(mp, "odp_pool", pool_entry)) { - ODP_ERR("Failed setting mempool operations\n"); + _ODP_ERR("Failed setting mempool operations\n"); goto fail; } @@ -430,7 +428,7 @@ static struct rte_mempool *mbuf_pool_create(const char *name, NULL, NULL); if (ret <= 0) { - ODP_ERR("Failed to populate mempool: %d\n", ret); + _ODP_ERR("Failed to populate mempool: %d\n", ret); goto fail; } @@ -439,8 +437,8 @@ static struct rte_mempool *mbuf_pool_create(const char *name, } if (populated != num) { - ODP_ERR("Failed to populate mempool with all requested blocks, populated: %u, " - "requested: %u\n", populated, num); + _ODP_ERR("Failed to populate mempool with all requested blocks, populated: %u, " + "requested: %u\n", populated, num); goto fail; } @@ -517,7 +515,7 @@ static unsigned pool_get_count(const struct rte_mempool *mp) odp_pool_info_t info; if (odp_pool_info(pool, &info)) { - ODP_ERR("Failed to read pool info\n"); + _ODP_ERR("Failed to read pool info\n"); return 0; } return info.params.pkt.num; @@ -566,7 +564,7 @@ static int pool_create(uint8_t *data, pool_t *pool) pkt_pool = mbuf_pool_create(pool_name, pool, mem_src_data->dpdk_elt_size); if (pkt_pool == NULL) { - ODP_ERR("Creating external DPDK pool failed\n"); + _ODP_ERR("Creating external DPDK pool failed\n"); return -1; } @@ -589,7 +587,7 @@ static void pool_obj_size(uint8_t *data, uint32_t *block_size, uint32_t *block_o if (odp_global_rw->dpdk_initialized == 0) { if (dpdk_pktio_init()) { - ODP_ERR("Initializing DPDK failed\n"); + _ODP_ERR("Initializing DPDK failed\n"); *block_size = 0; return; } @@ -641,8 +639,8 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, num = _odp_packet_alloc_multi(pool, max_len + frame_offset, pkt_table, mbuf_num); if (num != mbuf_num) { - ODP_DBG("_odp_packet_alloc_multi() unable to allocate all packets: " - "%d/%" PRIu16 " allocated\n", num, mbuf_num); + _ODP_DBG("_odp_packet_alloc_multi() unable to allocate all packets: " + "%d/%" PRIu16 " allocated\n", num, mbuf_num); for (i = num; i < mbuf_num; i++) rte_pktmbuf_free(mbuf_table[i]); } @@ -650,7 +648,7 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, for (i = 0; i < num; i++) { mbuf = mbuf_table[i]; if (odp_unlikely(mbuf->nb_segs != 1)) { - ODP_ERR("Segmented buffers not supported\n"); + _ODP_ERR("Segmented buffers not supported\n"); goto fail; } @@ -856,7 +854,7 @@ static inline int pkt_to_mbuf(pktio_entry_t *pktio_entry, if (odp_unlikely((rte_pktmbuf_alloc_bulk(pkt_dpdk->pkt_pool, mbuf_table, num)))) { - ODP_ERR("Failed to alloc mbuf\n"); + _ODP_ERR("Failed to alloc mbuf\n"); return 0; } for (i = 0; i < num; i++) { @@ -937,7 +935,7 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, mbuf = mbuf_table[i]; if (odp_unlikely(mbuf->nb_segs != 1)) { - ODP_ERR("Segmented buffers not supported\n"); + _ODP_ERR("Segmented buffers not supported\n"); rte_pktmbuf_free(mbuf); continue; } @@ -1078,7 +1076,7 @@ static uint32_t dpdk_vdev_mtu_get(uint16_t port_id) sockfd = socket(AF_INET, SOCK_DGRAM, 0); if (sockfd < 0) { - ODP_ERR("Failed to create control socket\n"); + _ODP_ERR("Failed to create control socket\n"); return 0; } @@ -1127,7 +1125,7 @@ static int dpdk_maxlen_set(pktio_entry_t *pktio_entry, uint32_t maxlen_input, ret = rte_eth_dev_set_mtu(pkt_dpdk->port_id, mtu); if (odp_unlikely(ret)) - ODP_ERR("rte_eth_dev_set_mtu() failed: %d\n", ret); + _ODP_ERR("rte_eth_dev_set_mtu() failed: %d\n", ret); pkt_dpdk->mtu = maxlen_input; pkt_dpdk->mtu_set = 1; @@ -1149,7 +1147,7 @@ static int dpdk_vdev_promisc_mode_get(uint16_t port_id) sockfd = socket(AF_INET, SOCK_DGRAM, 0); if (sockfd < 0) { - ODP_ERR("Failed to create control socket\n"); + _ODP_ERR("Failed to create control socket\n"); return -1; } @@ -1172,7 +1170,7 @@ static int dpdk_vdev_promisc_mode_set(uint16_t port_id, int enable) sockfd = socket(AF_INET, SOCK_DGRAM, 0); if (sockfd < 0) { - ODP_ERR("Failed to create control socket\n"); + _ODP_ERR("Failed to create control socket\n"); return -1; } @@ -1252,8 +1250,8 @@ static int dpdk_setup_eth_dev(pktio_entry_t *pktio_entry) pktio_entry->num_in_queue, pktio_entry->num_out_queue, ð_conf); if (ret < 0) { - ODP_ERR("Failed to setup device: err=%d, port=%" PRIu8 "\n", - ret, pkt_dpdk->port_id); + _ODP_ERR("Failed to setup device: err=%d, port=%" PRIu8 "\n", + ret, pkt_dpdk->port_id); return -1; } return 0; @@ -1302,7 +1300,7 @@ static int dpdk_pktio_init(void) i = pthread_getaffinity_np(pthread_self(), sizeof(original_cpuset), &original_cpuset); if (i != 0) { - ODP_ERR("Failed to read thread affinity: %d\n", i); + _ODP_ERR("Failed to read thread affinity: %d\n", i); return -1; } @@ -1316,7 +1314,7 @@ static int dpdk_pktio_init(void) masklen = odp_cpumask_to_str(&mask, mask_str, ODP_CPUMASK_STR_SIZE); if (masklen < 0) { - ODP_ERR("CPU mask error: %" PRId32 "\n", masklen); + _ODP_ERR("CPU mask error: %" PRId32 "\n", masklen); return -1; } @@ -1360,7 +1358,7 @@ static int dpdk_pktio_init(void) dpdk_argc = rte_strsplit(full_cmd, strlen(full_cmd), dpdk_argv, dpdk_argc, ' '); for (i = 0; i < dpdk_argc; ++i) - ODP_DBG("arg[%d]: %s\n", i, dpdk_argv[i]); + _ODP_DBG("arg[%d]: %s\n", i, dpdk_argv[i]); i = rte_eal_init(dpdk_argc, dpdk_argv); @@ -1368,22 +1366,22 @@ static int dpdk_pktio_init(void) optind = 0; if (i < 0) { - ODP_ERR("Cannot init the Intel DPDK EAL!\n"); + _ODP_ERR("Cannot init the Intel DPDK EAL!\n"); return -1; } else if (i + 1 != dpdk_argc) { - ODP_DBG("Some DPDK args were not processed!\n"); - ODP_DBG("Passed: %d Consumed %d\n", dpdk_argc, i + 1); + _ODP_DBG("Some DPDK args were not processed!\n"); + _ODP_DBG("Passed: %d Consumed %d\n", dpdk_argc, i + 1); } - ODP_DBG("rte_eal_init OK\n"); + _ODP_DBG("rte_eal_init OK\n"); rte_log_set_global_level(RTE_LOG_WARNING); i = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &original_cpuset); if (i) - ODP_ERR("Failed to reset thread affinity: %d\n", i); + _ODP_ERR("Failed to reset thread affinity: %d\n", i); - ODP_PRINT("\nDPDK version: %s\n", rte_version()); + _ODP_PRINT("\nDPDK version: %s\n", rte_version()); return 0; } @@ -1392,12 +1390,12 @@ static int dpdk_pktio_init(void) static int dpdk_pktio_init_global(void) { if (getenv("ODP_PKTIO_DISABLE_DPDK")) { - ODP_PRINT("PKTIO: dpdk pktio skipped," - " enabled export ODP_PKTIO_DISABLE_DPDK=1.\n"); + _ODP_PRINT("PKTIO: dpdk pktio skipped," + " enabled export ODP_PKTIO_DISABLE_DPDK=1.\n"); disable_pktio = 1; } else { - ODP_PRINT("PKTIO: initialized dpdk pktio," - " use export ODP_PKTIO_DISABLE_DPDK=1 to disable.\n"); + _ODP_PRINT("PKTIO: initialized dpdk pktio," + " use export ODP_PKTIO_DISABLE_DPDK=1 to disable.\n"); } return 0; } @@ -1408,7 +1406,7 @@ static int dpdk_pktio_init_local(void) cpu = sched_getcpu(); if (cpu < 0) { - ODP_ERR("getcpu failed\n"); + _ODP_ERR("getcpu failed\n"); return -1; } @@ -1458,33 +1456,33 @@ static void prepare_rss_conf(pktio_entry_t *pktio_entry, /* Print debug info about unsupported hash protocols */ if (p->hash_proto.proto.ipv4 && ((rss_hf_capa & ETH_RSS_IPV4) == 0)) - ODP_PRINT("DPDK: hash_proto.ipv4 not supported (rss_hf_capa 0x%" PRIx64 ")\n", - rss_hf_capa); + _ODP_PRINT("DPDK: hash_proto.ipv4 not supported (rss_hf_capa 0x%" PRIx64 ")\n", + rss_hf_capa); if (p->hash_proto.proto.ipv4_udp && ((rss_hf_capa & ETH_RSS_NONFRAG_IPV4_UDP) == 0)) - ODP_PRINT("DPDK: hash_proto.ipv4_udp not supported (rss_hf_capa 0x%" PRIx64 ")\n", - rss_hf_capa); + _ODP_PRINT("DPDK: hash_proto.ipv4_udp not supported (rss_hf_capa 0x%" PRIx64 ")\n", + rss_hf_capa); if (p->hash_proto.proto.ipv4_tcp && ((rss_hf_capa & ETH_RSS_NONFRAG_IPV4_TCP) == 0)) - ODP_PRINT("DPDK: hash_proto.ipv4_tcp not supported (rss_hf_capa 0x%" PRIx64 ")\n", - rss_hf_capa); + _ODP_PRINT("DPDK: hash_proto.ipv4_tcp not supported (rss_hf_capa 0x%" PRIx64 ")\n", + rss_hf_capa); if (p->hash_proto.proto.ipv6 && ((rss_hf_capa & ETH_RSS_IPV6) == 0)) - ODP_PRINT("DPDK: hash_proto.ipv6 not supported (rss_hf_capa 0x%" PRIx64 ")\n", - rss_hf_capa); + _ODP_PRINT("DPDK: hash_proto.ipv6 not supported (rss_hf_capa 0x%" PRIx64 ")\n", + rss_hf_capa); if (p->hash_proto.proto.ipv6_udp && ((rss_hf_capa & ETH_RSS_NONFRAG_IPV6_UDP) == 0)) - ODP_PRINT("DPDK: hash_proto.ipv6_udp not supported (rss_hf_capa 0x%" PRIx64 ")\n", - rss_hf_capa); + _ODP_PRINT("DPDK: hash_proto.ipv6_udp not supported (rss_hf_capa 0x%" PRIx64 ")\n", + rss_hf_capa); if (p->hash_proto.proto.ipv6_tcp && ((rss_hf_capa & ETH_RSS_NONFRAG_IPV6_TCP) == 0)) - ODP_PRINT("DPDK: hash_proto.ipv6_tcp not supported (rss_hf_capa 0x%" PRIx64 ")\n", - rss_hf_capa); + _ODP_PRINT("DPDK: hash_proto.ipv6_tcp not supported (rss_hf_capa 0x%" PRIx64 ")\n", + rss_hf_capa); hash_proto_to_rss_conf(&pkt_dpdk->rss_conf, &p->hash_proto); @@ -1531,7 +1529,7 @@ static int dpdk_output_queues_config(pktio_entry_t *pktio_entry, ret = rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info); if (ret) { - ODP_ERR("DPDK: rte_eth_dev_info_get() failed: %d\n", ret); + _ODP_ERR("DPDK: rte_eth_dev_info_get() failed: %d\n", ret); return -1; } @@ -1550,7 +1548,7 @@ static int dpdk_output_queues_config(pktio_entry_t *pktio_entry, if (num_tx_desc < dev_info.tx_desc_lim.nb_min || num_tx_desc > dev_info.tx_desc_lim.nb_max || num_tx_desc % dev_info.tx_desc_lim.nb_align) { - ODP_ERR("DPDK: invalid number of TX descriptors\n"); + _ODP_ERR("DPDK: invalid number of TX descriptors\n"); return -1; } pkt_dpdk->num_tx_desc[i] = num_tx_desc; @@ -1595,7 +1593,7 @@ static int dpdk_init_capability(pktio_entry_t *pktio_entry, if (ret == 0) { capa->set_op.op.mac_addr = 1; } else if (ret != -ENOTSUP && ret != -EPERM) { - ODP_ERR("Failed to set interface default MAC: %d\n", ret); + _ODP_ERR("Failed to set interface default MAC: %d\n", ret); return -1; } @@ -1744,7 +1742,7 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED, else if (dpdk_netdev_is_valid(netdev)) pkt_dpdk->port_id = atoi(netdev); else { - ODP_ERR("Invalid DPDK interface name: %s\n", netdev); + _ODP_ERR("Invalid DPDK interface name: %s\n", netdev); return -1; } @@ -1758,27 +1756,27 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED, pkt_dpdk->pool = pool; if (rte_eth_dev_count_avail() == 0) { - ODP_ERR("No DPDK ports found\n"); + _ODP_ERR("No DPDK ports found\n"); return -1; } memset(&dev_info, 0, sizeof(struct rte_eth_dev_info)); ret = rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info); if (ret) { - ODP_ERR("Failed to read device info: %d\n", ret); + _ODP_ERR("Failed to read device info: %d\n", ret); return -1; } /* Initialize runtime options */ if (init_options(pktio_entry, &dev_info)) { - ODP_ERR("Initializing runtime options failed\n"); + _ODP_ERR("Initializing runtime options failed\n"); return -1; } pkt_dpdk->flags.set_flow_hash = pkt_dpdk->opt.set_flow_hash; /* Copy for fast path access */ mtu = dpdk_mtu_get(pktio_entry); if (mtu == 0) { - ODP_ERR("Failed to read interface MTU\n"); + _ODP_ERR("Failed to read interface MTU\n"); return -1; } pkt_dpdk->mtu = mtu + _ODP_ETHHDR_LEN; @@ -1794,7 +1792,7 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED, /* Not supported by all PMDs, so ignore the return value */ if (ret) - ODP_DBG("Configuring multicast reception not supported by the PMD\n"); + _ODP_DBG("Configuring multicast reception not supported by the PMD\n"); /* Drivers requiring minimum burst size. Supports also *_vf versions * of the drivers. */ @@ -1825,7 +1823,7 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED, } } if (pkt_pool == NULL) { - ODP_ERR("Cannot init mbuf packet pool\n"); + _ODP_ERR("Cannot init mbuf packet pool\n"); return -1; } @@ -1843,7 +1841,7 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED, pkt_dpdk->mtu_max = RTE_MIN(pkt_dpdk->mtu_max, pkt_dpdk->data_room); if (dpdk_init_capability(pktio_entry, &dev_info)) { - ODP_ERR("Failed to initialize capability\n"); + _ODP_ERR("Failed to initialize capability\n"); return -1; } @@ -1871,8 +1869,7 @@ static int dpdk_setup_eth_tx(pktio_entry_t *pktio_entry, rte_eth_dev_socket_id(port_id), &dev_info->default_txconf); if (ret < 0) { - ODP_ERR("Queue setup failed: err=%d, port=%" PRIu8 "\n", - ret, port_id); + _ODP_ERR("Queue setup failed: err=%d, port=%" PRIu8 "\n", ret, port_id); return -1; } } @@ -1882,11 +1879,11 @@ static int dpdk_setup_eth_tx(pktio_entry_t *pktio_entry, for (i = 0; i < pktio_entry->num_out_queue && i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { ret = rte_eth_dev_set_tx_queue_stats_mapping(port_id, i, i); if (ret) { - ODP_DBG("Mapping per TX queue statistics not supported: %d\n", ret); + _ODP_DBG("Mapping per TX queue statistics not supported: %d\n", ret); break; } } - ODP_DBG("Mapped %" PRIu32 "/%d TX counters\n", i, RTE_ETHDEV_QUEUE_STAT_CNTRS); + _ODP_DBG("Mapped %" PRIu32 "/%d TX counters\n", i, RTE_ETHDEV_QUEUE_STAT_CNTRS); return 0; } @@ -1910,8 +1907,7 @@ static int dpdk_setup_eth_rx(const pktio_entry_t *pktio_entry, rte_eth_dev_socket_id(port_id), &rxconf, pkt_dpdk->pkt_pool); if (ret < 0) { - ODP_ERR("Queue setup failed: err=%d, port=%" PRIu8 "\n", - ret, port_id); + _ODP_ERR("Queue setup failed: err=%d, port=%" PRIu8 "\n", ret, port_id); return -1; } } @@ -1921,11 +1917,11 @@ static int dpdk_setup_eth_rx(const pktio_entry_t *pktio_entry, for (i = 0; i < pktio_entry->num_in_queue && i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { ret = rte_eth_dev_set_rx_queue_stats_mapping(port_id, i, i); if (ret) { - ODP_DBG("Mapping per RX queue statistics not supported: %d\n", ret); + _ODP_DBG("Mapping per RX queue statistics not supported: %d\n", ret); break; } } - ODP_DBG("Mapped %" PRIu32 "/%d RX counters\n", i, RTE_ETHDEV_QUEUE_STAT_CNTRS); + _ODP_DBG("Mapped %" PRIu32 "/%d RX counters\n", i, RTE_ETHDEV_QUEUE_STAT_CNTRS); return 0; } @@ -1947,7 +1943,7 @@ static int dpdk_start(pktio_entry_t *pktio_entry) /* Setup device */ if (dpdk_setup_eth_dev(pktio_entry)) { - ODP_ERR("Failed to configure device\n"); + _ODP_ERR("Failed to configure device\n"); return -1; } @@ -1963,16 +1959,15 @@ static int dpdk_start(pktio_entry_t *pktio_entry) if (pkt_dpdk->mtu_set && pktio_entry->capa.set_op.op.maxlen) { ret = dpdk_maxlen_set(pktio_entry, pkt_dpdk->mtu, 0); if (ret) { - ODP_ERR("Restoring device MTU failed: err=%d, port=%" PRIu8 "\n", - ret, port_id); + _ODP_ERR("Restoring device MTU failed: err=%d, port=%" PRIu8 "\n", + ret, port_id); return -1; } } /* Start device */ ret = rte_eth_dev_start(port_id); if (ret < 0) { - ODP_ERR("Device start failed: err=%d, port=%" PRIu8 "\n", - ret, port_id); + _ODP_ERR("Device start failed: err=%d, port=%" PRIu8 "\n", ret, port_id); return -1; } @@ -2192,7 +2187,7 @@ static int dpdk_link_info(pktio_entry_t *pktio_entry, odp_pktio_link_info_t *inf ret = rte_eth_dev_flow_ctrl_get(port_id, &fc_conf); if (ret && ret != -ENOTSUP) { - ODP_ERR("rte_eth_dev_flow_ctrl_get() failed\n"); + _ODP_ERR("rte_eth_dev_flow_ctrl_get() failed\n"); return -1; } @@ -2279,7 +2274,7 @@ static int dpdk_extra_stat_info(pktio_entry_t *pktio_entry, num_stats = rte_eth_xstats_get_names(port_id, NULL, 0); if (num_stats < 0) { - ODP_ERR("rte_eth_xstats_get_names() failed: %d\n", num_stats); + _ODP_ERR("rte_eth_xstats_get_names() failed: %d\n", num_stats); return num_stats; } else if (info == NULL || num == 0 || num_stats == 0) { return num_stats; @@ -2289,7 +2284,7 @@ static int dpdk_extra_stat_info(pktio_entry_t *pktio_entry, ret = rte_eth_xstats_get_names(port_id, xstats_names, num_stats); if (ret < 0 || ret > num_stats) { - ODP_ERR("rte_eth_xstats_get_names() failed: %d\n", ret); + _ODP_ERR("rte_eth_xstats_get_names() failed: %d\n", ret); return -1; } num_stats = ret; @@ -2309,7 +2304,7 @@ static int dpdk_extra_stats(pktio_entry_t *pktio_entry, num_stats = rte_eth_xstats_get(port_id, NULL, 0); if (num_stats < 0) { - ODP_ERR("rte_eth_xstats_get() failed: %d\n", num_stats); + _ODP_ERR("rte_eth_xstats_get() failed: %d\n", num_stats); return num_stats; } else if (stats == NULL || num == 0 || num_stats == 0) { return num_stats; @@ -2319,7 +2314,7 @@ static int dpdk_extra_stats(pktio_entry_t *pktio_entry, ret = rte_eth_xstats_get(port_id, xstats, num_stats); if (ret < 0 || ret > num_stats) { - ODP_ERR("rte_eth_xstats_get() failed: %d\n", ret); + _ODP_ERR("rte_eth_xstats_get() failed: %d\n", ret); return -1; } num_stats = ret; @@ -2339,7 +2334,7 @@ static int dpdk_extra_stat_counter(pktio_entry_t *pktio_entry, uint32_t id, ret = rte_eth_xstats_get_by_id(port_id, &xstat_id, stat, 1); if (ret != 1) { - ODP_ERR("rte_eth_xstats_get_by_id() failed: %d\n", ret); + _ODP_ERR("rte_eth_xstats_get_by_id() failed: %d\n", ret); return -1; } @@ -2353,14 +2348,13 @@ static int dpdk_pktin_stats(pktio_entry_t *pktio_entry, uint32_t index, int ret; if (odp_unlikely(index > RTE_ETHDEV_QUEUE_STAT_CNTRS - 1)) { - ODP_ERR("DPDK supports max %d per queue counters\n", - RTE_ETHDEV_QUEUE_STAT_CNTRS); + _ODP_ERR("DPDK supports max %d per queue counters\n", RTE_ETHDEV_QUEUE_STAT_CNTRS); return -1; } ret = rte_eth_stats_get(pkt_priv(pktio_entry)->port_id, &rte_stats); if (odp_unlikely(ret)) { - ODP_ERR("Failed to read DPDK pktio stats: %d\n", ret); + _ODP_ERR("Failed to read DPDK pktio stats: %d\n", ret); return -1; } @@ -2380,14 +2374,13 @@ static int dpdk_pktout_stats(pktio_entry_t *pktio_entry, uint32_t index, int ret; if (odp_unlikely(index > RTE_ETHDEV_QUEUE_STAT_CNTRS - 1)) { - ODP_ERR("DPDK supports max %d per queue counters\n", - RTE_ETHDEV_QUEUE_STAT_CNTRS); + _ODP_ERR("DPDK supports max %d per queue counters\n", RTE_ETHDEV_QUEUE_STAT_CNTRS); return -1; } ret = rte_eth_stats_get(pkt_priv(pktio_entry)->port_id, &rte_stats); if (odp_unlikely(ret)) { - ODP_ERR("Failed to read DPDK pktio stats: %d\n", ret); + _ODP_ERR("Failed to read DPDK pktio stats: %d\n", ret); return -1; } diff --git a/platform/linux-generic/pktio/ethtool_rss.c b/platform/linux-generic/pktio/ethtool_rss.c index 80a66420e..df97e2417 100644 --- a/platform/linux-generic/pktio/ethtool_rss.c +++ b/platform/linux-generic/pktio/ethtool_rss.c @@ -248,6 +248,6 @@ void _odp_rss_conf_print(const odp_pktin_hash_proto_t *hash_proto) " IPV6 UDP\n"); str[len] = '\0'; - ODP_PRINT("%s\n", str); + _ODP_PRINT("%s\n", str); } diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c index f3246c068..586bc0aa7 100644 --- a/platform/linux-generic/pktio/ipc.c +++ b/platform/linux-generic/pktio/ipc.c @@ -128,7 +128,7 @@ static ring_ptr_t *_ring_create(const char *name, uint32_t count, /* count must be a power of 2 */ if (!_ODP_CHECK_IS_POWER2(count)) { - ODP_ERR("Requested size is invalid, must be a power of 2\n"); + _ODP_ERR("Requested size is invalid, must be a power of 2\n"); _odp_errno = EINVAL; return NULL; } @@ -145,7 +145,7 @@ static ring_ptr_t *_ring_create(const char *name, uint32_t count, } else { _odp_errno = ENOMEM; - ODP_ERR("Cannot reserve memory\n"); + _ODP_ERR("Cannot reserve memory\n"); } return r; @@ -210,8 +210,7 @@ static int _ipc_master_start(pktio_entry_t *pktio_entry) shm = _ipc_map_remote_pool(pinfo->slave.pool_name, pinfo->slave.pid); if (shm == ODP_SHM_INVALID) { - ODP_DBG("no pool file %s for pid %d\n", - pinfo->slave.pool_name, pinfo->slave.pid); + _ODP_DBG("no pool file %s for pid %d\n", pinfo->slave.pool_name, pinfo->slave.pid); return -1; } @@ -238,7 +237,7 @@ static int _ipc_init_master(pktio_entry_t *pktio_entry, uint32_t ring_mask; if ((uint64_t)_ODP_ROUNDUP_POWER2_U32(pool->num + 1) > UINT32_MAX) { - ODP_ERR("Too large packet pool\n"); + _ODP_ERR("Too large packet pool\n"); return -1; } @@ -254,13 +253,13 @@ static int _ipc_init_master(pktio_entry_t *pktio_entry, pktio_ipc->ring_mask = ring_mask; if (strlen(dev) > (ODP_POOL_NAME_LEN - sizeof("_m_prod"))) { - ODP_ERR("too big ipc name\n"); + _ODP_ERR("too big ipc name\n"); return -1; } pktio_ipc->rx.cache = _ring_create("ipc_rx_cache", ring_size, 0); if (!pktio_ipc->rx.cache) { - ODP_ERR("pid %d unable to create ipc rx cache\n", getpid()); + _ODP_ERR("pid %d unable to create ipc rx cache\n", getpid()); return -1; } @@ -271,13 +270,12 @@ static int _ipc_init_master(pktio_entry_t *pktio_entry, pktio_ipc->tx.send = _ring_create(ipc_shm_name, ring_size, ODP_SHM_PROC | ODP_SHM_EXPORT); if (!pktio_ipc->tx.send) { - ODP_ERR("pid %d unable to create ipc ring %s name\n", - getpid(), ipc_shm_name); + _ODP_ERR("pid %d unable to create ipc ring %s name\n", getpid(), ipc_shm_name); return -1; } - ODP_DBG("Created IPC ring: %s, count %d, free %d\n", - ipc_shm_name, _ring_count(pktio_ipc->tx.send, ring_mask), - _ring_free_count(pktio_ipc->tx.send, ring_mask)); + _ODP_DBG("Created IPC ring: %s, count %d, free %d\n", + ipc_shm_name, _ring_count(pktio_ipc->tx.send, ring_mask), + _ring_free_count(pktio_ipc->tx.send, ring_mask)); /* generate name in shm like ipc_pktio_p for * already processed packets @@ -286,44 +284,41 @@ static int _ipc_init_master(pktio_entry_t *pktio_entry, pktio_ipc->tx.free = _ring_create(ipc_shm_name, ring_size, ODP_SHM_PROC | ODP_SHM_EXPORT); if (!pktio_ipc->tx.free) { - ODP_ERR("pid %d unable to create ipc ring %s name\n", - getpid(), ipc_shm_name); + _ODP_ERR("pid %d unable to create ipc ring %s name\n", getpid(), ipc_shm_name); goto free_m_prod; } - ODP_DBG("Created IPC ring: %s, count %d, free %d\n", - ipc_shm_name, _ring_count(pktio_ipc->tx.free, ring_mask), - _ring_free_count(pktio_ipc->tx.free, ring_mask)); + _ODP_DBG("Created IPC ring: %s, count %d, free %d\n", + ipc_shm_name, _ring_count(pktio_ipc->tx.free, ring_mask), + _ring_free_count(pktio_ipc->tx.free, ring_mask)); snprintf(ipc_shm_name, sizeof(ipc_shm_name), "%s_s_prod", dev); pktio_ipc->rx.recv = _ring_create(ipc_shm_name, ring_size, ODP_SHM_PROC | ODP_SHM_EXPORT); if (!pktio_ipc->rx.recv) { - ODP_ERR("pid %d unable to create ipc ring %s name\n", - getpid(), ipc_shm_name); + _ODP_ERR("pid %d unable to create ipc ring %s name\n", getpid(), ipc_shm_name); goto free_m_cons; } - ODP_DBG("Created IPC ring: %s, count %d, free %d\n", - ipc_shm_name, _ring_count(pktio_ipc->rx.recv, ring_mask), - _ring_free_count(pktio_ipc->rx.recv, ring_mask)); + _ODP_DBG("Created IPC ring: %s, count %d, free %d\n", + ipc_shm_name, _ring_count(pktio_ipc->rx.recv, ring_mask), + _ring_free_count(pktio_ipc->rx.recv, ring_mask)); snprintf(ipc_shm_name, sizeof(ipc_shm_name), "%s_s_cons", dev); pktio_ipc->rx.free = _ring_create(ipc_shm_name, ring_size, ODP_SHM_PROC | ODP_SHM_EXPORT); if (!pktio_ipc->rx.free) { - ODP_ERR("pid %d unable to create ipc ring %s name\n", - getpid(), ipc_shm_name); + _ODP_ERR("pid %d unable to create ipc ring %s name\n", getpid(), ipc_shm_name); goto free_s_prod; } - ODP_DBG("Created IPC ring: %s, count %d, free %d\n", - ipc_shm_name, _ring_count(pktio_ipc->rx.free, ring_mask), - _ring_free_count(pktio_ipc->rx.free, ring_mask)); + _ODP_DBG("Created IPC ring: %s, count %d, free %d\n", + ipc_shm_name, _ring_count(pktio_ipc->rx.free, ring_mask), + _ring_free_count(pktio_ipc->rx.free, ring_mask)); /* Set up pool name for remote info */ pinfo = pktio_ipc->pinfo; pool_name = _ipc_odp_buffer_pool_shm_name(pool_hdl); if (strlen(pool_name) > ODP_POOL_NAME_LEN) { - ODP_ERR("pid %d ipc pool name %s is too big %zu\n", - getpid(), pool_name, strlen(pool_name)); + _ODP_ERR("pid %d ipc pool name %s is too big %zu\n", + getpid(), pool_name, strlen(pool_name)); goto free_s_prod; } @@ -340,7 +335,7 @@ static int _ipc_init_master(pktio_entry_t *pktio_entry, pktio_ipc->pool = pool_hdl; - ODP_DBG("Pre init... DONE.\n"); + _ODP_DBG("Pre init... DONE.\n"); pinfo->master.init_done = 1; _ipc_master_start(pktio_entry); @@ -378,7 +373,7 @@ static odp_shm_t _ipc_map_remote_pool(const char *name, int pid) snprintf(rname, ODP_SHM_NAME_LEN, "remote-%s", name); shm = odp_shm_import(name, pid, rname); if (shm == ODP_SHM_INVALID) { - ODP_ERR("unable map %s\n", name); + _ODP_ERR("unable map %s\n", name); return ODP_SHM_INVALID; } @@ -392,7 +387,7 @@ static void *_ipc_shm_map(char *name, int pid) shm = odp_shm_import(name, pid, name); if (ODP_SHM_INVALID == shm) { - ODP_ERR("unable to map: %s\n", name); + _ODP_ERR("unable to map: %s\n", name); return NULL; } @@ -407,20 +402,20 @@ static int _ipc_init_slave(const char *dev, pktio_entry_t *pktio_entry, uint32_t ring_size = pktio_ipc->pinfo->master.ring_size; if (strlen(dev) > (ODP_POOL_NAME_LEN - sizeof("_slave_r"))) { - ODP_ERR("Too big ipc name\n"); + _ODP_ERR("Too big ipc name\n"); return -1; } /* Check that IPC rings are able to store all packets */ if (pool->num >= ring_size) { - ODP_ERR("Slave process packet pool too large. Master process " + _ODP_ERR("Slave process packet pool too large. Master process " "packet pool has to be larger than slave pool.\n"); return -1; } pktio_ipc->rx.cache = _ring_create("ipc_rx_cache", ring_size, 0); if (!pktio_ipc->rx.cache) { - ODP_ERR("Pid %d unable to create ipc rx cache\n", getpid()); + _ODP_ERR("Pid %d unable to create ipc rx cache\n", getpid()); return -1; } pktio_ipc->ring_size = ring_size; @@ -442,7 +437,7 @@ static int _ipc_slave_start(pktio_entry_t *pktio_entry) uint32_t ring_mask = pktio_ipc->ring_mask; if (sscanf(pktio_entry->name, "ipc:%d:%s", &pid, tail) != 2) { - ODP_ERR("wrong pktio name\n"); + _ODP_ERR("wrong pktio name\n"); return -1; } @@ -451,47 +446,43 @@ static int _ipc_slave_start(pktio_entry_t *pktio_entry) snprintf(ipc_shm_name, sizeof(ipc_shm_name), "%s_m_prod", dev); pktio_ipc->rx.recv = _ipc_shm_map(ipc_shm_name, pid); if (!pktio_ipc->rx.recv) { - ODP_DBG("pid %d unable to find ipc ring %s name\n", - getpid(), dev); + _ODP_DBG("pid %d unable to find ipc ring %s name\n", getpid(), dev); sleep(1); return -1; } - ODP_DBG("Connected IPC ring: %s, count %d, free %d\n", - ipc_shm_name, _ring_count(pktio_ipc->rx.recv, ring_mask), - _ring_free_count(pktio_ipc->rx.recv, ring_mask)); + _ODP_DBG("Connected IPC ring: %s, count %d, free %d\n", + ipc_shm_name, _ring_count(pktio_ipc->rx.recv, ring_mask), + _ring_free_count(pktio_ipc->rx.recv, ring_mask)); snprintf(ipc_shm_name, sizeof(ipc_shm_name), "%s_m_cons", dev); pktio_ipc->rx.free = _ipc_shm_map(ipc_shm_name, pid); if (!pktio_ipc->rx.free) { - ODP_ERR("pid %d unable to find ipc ring %s name\n", - getpid(), dev); + _ODP_ERR("pid %d unable to find ipc ring %s name\n", getpid(), dev); goto free_m_prod; } - ODP_DBG("Connected IPC ring: %s, count %d, free %d\n", - ipc_shm_name, _ring_count(pktio_ipc->rx.free, ring_mask), - _ring_free_count(pktio_ipc->rx.free, ring_mask)); + _ODP_DBG("Connected IPC ring: %s, count %d, free %d\n", + ipc_shm_name, _ring_count(pktio_ipc->rx.free, ring_mask), + _ring_free_count(pktio_ipc->rx.free, ring_mask)); snprintf(ipc_shm_name, sizeof(ipc_shm_name), "%s_s_prod", dev); pktio_ipc->tx.send = _ipc_shm_map(ipc_shm_name, pid); if (!pktio_ipc->tx.send) { - ODP_ERR("pid %d unable to find ipc ring %s name\n", - getpid(), dev); + _ODP_ERR("pid %d unable to find ipc ring %s name\n", getpid(), dev); goto free_m_cons; } - ODP_DBG("Connected IPC ring: %s, count %d, free %d\n", - ipc_shm_name, _ring_count(pktio_ipc->tx.send, ring_mask), - _ring_free_count(pktio_ipc->tx.send, ring_mask)); + _ODP_DBG("Connected IPC ring: %s, count %d, free %d\n", + ipc_shm_name, _ring_count(pktio_ipc->tx.send, ring_mask), + _ring_free_count(pktio_ipc->tx.send, ring_mask)); snprintf(ipc_shm_name, sizeof(ipc_shm_name), "%s_s_cons", dev); pktio_ipc->tx.free = _ipc_shm_map(ipc_shm_name, pid); if (!pktio_ipc->tx.free) { - ODP_ERR("pid %d unable to find ipc ring %s name\n", - getpid(), dev); + _ODP_ERR("pid %d unable to find ipc ring %s name\n", getpid(), dev); goto free_s_prod; } - ODP_DBG("Connected IPC ring: %s, count %d, free %d\n", - ipc_shm_name, _ring_count(pktio_ipc->tx.free, ring_mask), - _ring_free_count(pktio_ipc->tx.free, ring_mask)); + _ODP_DBG("Connected IPC ring: %s, count %d, free %d\n", + ipc_shm_name, _ring_count(pktio_ipc->tx.free, ring_mask), + _ring_free_count(pktio_ipc->tx.free, ring_mask)); /* Get info about remote pool */ pinfo = pktio_ipc->pinfo; @@ -506,7 +497,7 @@ static int _ipc_slave_start(pktio_entry_t *pktio_entry) odp_atomic_store_u32(&pktio_ipc->ready, 1); pinfo->slave.init_done = 1; - ODP_DBG("%s started.\n", pktio_entry->name); + _ODP_DBG("%s started.\n", pktio_entry->name); return 0; free_s_prod: @@ -560,7 +551,7 @@ static int ipc_pktio_open(odp_pktio_t id ODP_UNUSED, } pktio_ipc->pinfo = pinfo; pktio_ipc->pinfo_shm = shm; - ODP_DBG("process %d is slave\n", getpid()); + _ODP_DBG("process %d is slave\n", getpid()); ret = _ipc_init_slave(name, pktio_entry, pool); } else { pktio_ipc->type = PKTIO_TYPE_IPC_MASTER; @@ -569,7 +560,7 @@ static int ipc_pktio_open(odp_pktio_t id ODP_UNUSED, ODP_CACHE_LINE_SIZE, ODP_SHM_EXPORT | ODP_SHM_SINGLE_VA); if (ODP_SHM_INVALID == shm) { - ODP_ERR("can not create shm %s\n", name); + _ODP_ERR("can not create shm %s\n", name); return -1; } @@ -579,7 +570,7 @@ static int ipc_pktio_open(odp_pktio_t id ODP_UNUSED, pktio_ipc->pinfo = pinfo; pktio_ipc->pinfo_shm = shm; - ODP_DBG("process %d is master\n", getpid()); + _ODP_DBG("process %d is master\n", getpid()); ret = _ipc_init_master(pktio_entry, dev, pool); } @@ -649,7 +640,7 @@ static int ipc_pktio_recv_lockless(pktio_entry_t *pktio_entry, r = pktio_ipc->rx.cache; pkts = ring_ptr_deq_multi(r, ring_mask, ipcbufs_p, len); if (odp_unlikely(pkts < 0)) - ODP_ABORT("internal error dequeue\n"); + _ODP_ABORT("internal error dequeue\n"); /* rx from other app */ if (pkts == 0) { @@ -658,7 +649,7 @@ static int ipc_pktio_recv_lockless(pktio_entry_t *pktio_entry, pkts = ring_ptr_deq_multi(r, ring_mask, ipcbufs_p, len); if (odp_unlikely(pkts < 0)) - ODP_ABORT("internal error dequeue\n"); + _ODP_ABORT("internal error dequeue\n"); } /* fast path */ @@ -678,7 +669,7 @@ static int ipc_pktio_recv_lockless(pktio_entry_t *pktio_entry, pool = pktio_ipc->pool; if (odp_unlikely(pool == ODP_POOL_INVALID)) - ODP_ABORT("invalid pool"); + _ODP_ABORT("invalid pool"); data_pool_off = (uint8_t *)phdr->seg_data - (uint8_t *)pktio_ipc->remote_base_addr; @@ -700,8 +691,8 @@ static int ipc_pktio_recv_lockless(pktio_entry_t *pktio_entry, /* Copy packet data. */ pkt_data = odp_packet_data(pkt); if (odp_unlikely(!pkt_data)) - ODP_ABORT("unable to map pkt_data ipc_slave %d\n", - (PKTIO_TYPE_IPC_SLAVE == pktio_ipc->type)); + _ODP_ABORT("unable to map pkt_data ipc_slave %d\n", + (PKTIO_TYPE_IPC_SLAVE == pktio_ipc->type)); /* Copy packet data from shared pool to local pool. */ rmt_data_ptr = (uint8_t *)pktio_ipc->pool_mdata_base + @@ -799,7 +790,7 @@ static int ipc_pktio_send_lockless(pktio_entry_t *pktio_entry, newpkt = odp_packet_copy(pkt, pktio_ipc->pool); if (newpkt == ODP_PACKET_INVALID) - ODP_ABORT("Unable to copy packet\n"); + _ODP_ABORT("Unable to copy packet\n"); odp_packet_free(pkt); pkt_table_mapped[i] = newpkt; @@ -870,7 +861,7 @@ static int ipc_start(pktio_entry_t *pktio_entry) uint32_t ready = odp_atomic_load_u32(&pktio_ipc->ready); if (ready) { - ODP_ABORT("%s Already started\n", pktio_entry->name); + _ODP_ABORT("%s Already started\n", pktio_entry->name); return -1; } diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index 695bc477e..947edd476 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -106,7 +106,7 @@ static int loopback_queue_destroy(odp_queue_t queue) } while (event != ODP_EVENT_INVALID); if (odp_queue_destroy(queue)) { - ODP_ERR("Destroying loopback pktio queue failed\n"); + _ODP_ERR("Destroying loopback pktio queue failed\n"); return -1; } return 0; @@ -131,7 +131,7 @@ static int loopback_pktout_queue_config(pktio_entry_t *pktio_entry, pkt_loop->loopq = odp_queue_create(queue_name, &queue_param); if (pkt_loop->loopq == ODP_QUEUE_INVALID) { - ODP_ERR("Creating loopback pktio queue failed\n"); + _ODP_ERR("Creating loopback pktio queue failed\n"); return -1; } @@ -408,7 +408,7 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pktio_entry->stats.out_packets += ret; pktio_entry->stats.out_octets += out_octets_tbl[ret - 1]; } else { - ODP_DBG("queue enqueue failed %i\n", ret); + _ODP_DBG("queue enqueue failed %i\n", ret); ret = -1; } @@ -468,7 +468,7 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry) odp_queue_capability_t queue_capa; if (odp_queue_capability(&queue_capa)) { - ODP_ERR("Queue capability failed\n"); + _ODP_ERR("Queue capability failed\n"); return -1; } @@ -579,7 +579,7 @@ static int loopback_pktout_stats(pktio_entry_t *pktio_entry, static int loop_init_global(void) { - ODP_PRINT("PKTIO: initialized loop interface.\n"); + _ODP_PRINT("PKTIO: initialized loop interface.\n"); return 0; } diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index 6cb3feeb5..41ded2ae1 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -142,8 +142,7 @@ static int lookup_opt(const char *opt_name, const char *drv_name, int *val) ret = _odp_libconfig_lookup_ext_int(base, drv_name, opt_name, val); if (ret == 0) - ODP_ERR("Unable to find netmap configuration option: %s\n", - opt_name); + _ODP_ERR("Unable to find netmap configuration option: %s\n", opt_name); return ret; } @@ -157,7 +156,7 @@ static int init_options(pktio_entry_t *pktio_entry) return -1; if (opt->nr_rx_slots < 0 || opt->nr_rx_slots > 4096) { - ODP_ERR("Invalid number of RX slots\n"); + _ODP_ERR("Invalid number of RX slots\n"); return -1; } @@ -166,13 +165,13 @@ static int init_options(pktio_entry_t *pktio_entry) return -1; if (opt->nr_tx_slots < 0 || opt->nr_tx_slots > 4096) { - ODP_ERR("Invalid number of TX slots\n"); + _ODP_ERR("Invalid number of TX slots\n"); return -1; } - ODP_DBG("netmap interface: %s\n", pkt_priv(pktio_entry)->if_name); - ODP_DBG(" num_rx_desc: %d\n", opt->nr_rx_slots); - ODP_DBG(" num_tx_desc: %d\n", opt->nr_tx_slots); + _ODP_DBG("netmap interface: %s\n", pkt_priv(pktio_entry)->if_name); + _ODP_DBG(" num_rx_desc: %d\n", opt->nr_rx_slots); + _ODP_DBG(" num_tx_desc: %d\n", opt->nr_tx_slots); return 0; } @@ -220,7 +219,7 @@ static int netmap_do_ioctl(pktio_entry_t *pktio_entry, unsigned long cmd, } done: if (err) - ODP_ERR("ioctl err %d %lu: %s\n", err, cmd, strerror(errno)); + _ODP_ERR("ioctl err %d %lu: %s\n", err, cmd, strerror(errno)); return err; } @@ -246,7 +245,7 @@ static inline void map_netmap_rings(netmap_ring_t *rings, remainder = num_rings % num_queues; if (remainder) - ODP_DBG("WARNING: Netmap rings mapped unevenly to queues\n"); + _ODP_DBG("WARNING: Netmap rings mapped unevenly to queues\n"); for (i = 0; i < num_queues; i++) { desc_ring = &rings[i]; @@ -283,7 +282,7 @@ static int netmap_input_queues_config(pktio_entry_t *pktio_entry, if (_odp_rss_conf_set_fd(pkt_priv(pktio_entry)->sockfd, pkt_priv(pktio_entry)->if_name, &p->hash_proto)) { - ODP_ERR("Failed to configure input hash\n"); + _ODP_ERR("Failed to configure input hash\n"); return -1; } } @@ -342,7 +341,7 @@ static int netmap_close(pktio_entry_t *pktio_entry) if (pkt_nm->sockfd != -1 && close(pkt_nm->sockfd) != 0) { _odp_errno = errno; - ODP_ERR("close(sockfd): %s\n", strerror(errno)); + _ODP_ERR("close(sockfd): %s\n", strerror(errno)); return -1; } return 0; @@ -407,7 +406,7 @@ static inline int netmap_wait_for_link(pktio_entry_t *pktio_entry) if (ret == 1) return 1; } - ODP_DBG("%s link is down\n", pkt_priv(pktio_entry)->if_name); + _ODP_DBG("%s link is down\n", pkt_priv(pktio_entry)->if_name); return 0; } @@ -430,7 +429,7 @@ static void netmap_init_capability(pktio_entry_t *pktio_entry) /* Have to use a single descriptor to fetch packets from all * netmap rings */ capa->max_input_queues = 1; - ODP_DBG("Unable to store all %" PRIu32 " rx rings (max %d)\n" + _ODP_DBG("Unable to store all %" PRIu32 " rx rings (max %d)\n" " max input queues: %u\n", pkt_nm->num_rx_rings, NM_MAX_DESC, capa->max_input_queues); } @@ -440,7 +439,7 @@ static void netmap_init_capability(pktio_entry_t *pktio_entry) capa->max_output_queues = pkt_nm->num_tx_rings; if (capa->max_output_queues > NM_MAX_DESC) { capa->max_output_queues = NM_MAX_DESC; - ODP_DBG("Unable to store all %" PRIu32 " tx rings (max %d)\n" + _ODP_DBG("Unable to store all %" PRIu32 " tx rings (max %d)\n" " max output queues: %u\n", pkt_nm->num_tx_rings, NM_MAX_DESC, capa->max_output_queues); } @@ -525,14 +524,14 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, /* Initialize runtime options */ if (init_options(pktio_entry)) { - ODP_ERR("Initializing runtime options failed\n"); + _ODP_ERR("Initializing runtime options failed\n"); return -1; } /* Read netmap buffer size */ nm_buf_size = read_netmap_buf_size(); if (!nm_buf_size) { - ODP_ERR("Unable to read netmap buf size\n"); + _ODP_ERR("Unable to read netmap buf size\n"); return -1; } pkt_nm->mtu_max = _ODP_SOCKET_MTU_MAX; @@ -542,7 +541,7 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, if (!pkt_nm->is_virtual) { sockfd = socket(AF_INET, SOCK_DGRAM, 0); if (sockfd == -1) { - ODP_ERR("Cannot get device control socket\n"); + _ODP_ERR("Cannot get device control socket\n"); return -1; } pkt_nm->sockfd = sockfd; @@ -551,7 +550,7 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, * whichever is smaller. */ mtu = _odp_mtu_get_fd(pkt_nm->sockfd, pkt_nm->if_name); if (mtu == 0) { - ODP_ERR("Unable to read interface MTU\n"); + _ODP_ERR("Unable to read interface MTU\n"); goto error; } pkt_nm->mtu = (mtu < nm_buf_size) ? mtu : nm_buf_size; @@ -560,7 +559,7 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, if (mtu > nm_buf_size) { if (_odp_mtu_set_fd(pkt_nm->sockfd, pkt_nm->if_name, nm_buf_size)) { - ODP_ERR("Unable to set interface MTU\n"); + _ODP_ERR("Unable to set interface MTU\n"); goto error; } } @@ -569,7 +568,7 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, * capability info. */ desc = nm_open(pkt_nm->nm_name, NULL, 0, NULL); if (desc == NULL) { - ODP_ERR("nm_open(%s) failed\n", pkt_nm->nm_name); + _ODP_ERR("nm_open(%s) failed\n", pkt_nm->nm_name); goto error; } pkt_nm->num_rx_rings = desc->nifp->ni_rx_rings; @@ -589,7 +588,7 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, uint32_t tid = syscall(SYS_gettid); if ((int)tid == -1) - ODP_DBG("Unable to fetch thread ID. VALE port MAC " + _ODP_DBG("Unable to fetch thread ID. VALE port MAC " "addresses may not be unique.\n"); pktio_entry->capa.max_input_queues = 1; @@ -611,7 +610,7 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, /* Check if RSS is supported. If not, set 'max_input_queues' to 1. */ if (_odp_rss_conf_get_supported_fd(pkt_nm->sockfd, netdev, &hash_proto) == 0) { - ODP_DBG("RSS not supported\n"); + _ODP_DBG("RSS not supported\n"); pktio_entry->capa.max_input_queues = 1; } @@ -619,7 +618,7 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, if (err) goto error; if ((pkt_nm->if_flags & IFF_UP) == 0) - ODP_DBG("%s is down\n", pkt_nm->if_name); + _ODP_DBG("%s is down\n", pkt_nm->if_name); err = _odp_mac_addr_get_fd(pkt_nm->sockfd, netdev, pkt_nm->if_mac); if (err) @@ -628,8 +627,8 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, /* netmap uses only ethtool to get statistics counters */ err = _odp_ethtool_stats_get_fd(pkt_nm->sockfd, pkt_nm->if_name, &cur_stats); if (err) { - ODP_DBG("netmap pktio %s does not support statistics counters\n", - pkt_nm->if_name); + _ODP_DBG("netmap pktio %s does not support statistics counters\n", + pkt_nm->if_name); pktio_entry->stats_type = STATS_UNSUPPORTED; } else { pktio_entry->stats_type = STATS_ETHTOOL; @@ -703,7 +702,7 @@ static int netmap_start(pktio_entry_t *pktio_entry) /* Use nm_open() to parse netmap flags from interface name */ desc_ptr = nm_open(pkt_nm->nm_name, NULL, 0, NULL); if (desc_ptr == NULL) { - ODP_ERR("nm_start(%s) failed\n", pkt_nm->nm_name); + _ODP_ERR("nm_start(%s) failed\n", pkt_nm->nm_name); goto error; } struct nm_desc base_desc = *desc_ptr; @@ -736,7 +735,7 @@ static int netmap_start(pktio_entry_t *pktio_entry) desc_ring[0].desc[0] = nm_open(pkt_nm->nm_name, NULL, flags, &base_desc); if (desc_ring[0].desc[0] == NULL) { - ODP_ERR("nm_start(%s) failed\n", pkt_nm->nm_name); + _ODP_ERR("nm_start(%s) failed\n", pkt_nm->nm_name); goto error; } /* Open rest of the rx descriptors (one per netmap ring) */ @@ -755,8 +754,7 @@ static int netmap_start(pktio_entry_t *pktio_entry) desc_ring[i].desc[j] = nm_open(pkt_nm->nm_name, NULL, flags, &base_desc); if (desc_ring[i].desc[j] == NULL) { - ODP_ERR("nm_start(%s) failed\n", - pkt_nm->nm_name); + _ODP_ERR("nm_start(%s) failed\n", pkt_nm->nm_name); goto error; } } @@ -778,8 +776,7 @@ static int netmap_start(pktio_entry_t *pktio_entry) desc_ring[i].desc[j] = nm_open(pkt_nm->nm_name, NULL, flags, &base_desc); if (desc_ring[i].desc[j] == NULL) { - ODP_ERR("nm_start(%s) failed\n", - pkt_nm->nm_name); + _ODP_ERR("nm_start(%s) failed\n", pkt_nm->nm_name); goto error; } } @@ -934,7 +931,7 @@ static inline int netmap_recv_desc(pktio_entry_t *pktio_entry, slot_tbl[num_rx].len = ring->slot[slot_id].len; num_rx++; } else { - ODP_DBG("Dropped oversized packet: %" PRIu16 " " + _ODP_DBG("Dropped oversized packet: %" PRIu16 " " "B\n", ring->slot[slot_id].len); } ring->cur = nm_ring_next(ring, slot_id); @@ -1031,7 +1028,7 @@ static int netmap_recv(pktio_entry_t *pktio_entry, int index, struct timeval tout = {.tv_sec = 0, .tv_usec = 0}; if (select(max_fd + 1, &empty_rings, NULL, NULL, &tout) == -1) - ODP_ERR("RX: select error\n"); + _ODP_ERR("RX: select error\n"); } if (!pkt_nm->lockless_rx) odp_ticketlock_unlock(&pkt_nm->rx_desc_ring[index].lock); @@ -1293,11 +1290,11 @@ static void netmap_print(pktio_entry_t *pktio_entry) static int netmap_init_global(void) { if (getenv("ODP_PKTIO_DISABLE_NETMAP")) { - ODP_PRINT("PKTIO: netmap pktio skipped," + _ODP_PRINT("PKTIO: netmap pktio skipped," " enabled export ODP_PKTIO_DISABLE_NETMAP=1.\n"); disable_pktio = 1; } else { - ODP_PRINT("PKTIO: initialized netmap pktio," + _ODP_PRINT("PKTIO: initialized netmap pktio," " use export ODP_PKTIO_DISABLE_NETMAP=1 to disable.\n" " Netmap prefixes are netmap:eth0 or vale:eth0. Refer to" " Netmap documentation for usage information.\n"); diff --git a/platform/linux-generic/pktio/null.c b/platform/linux-generic/pktio/null.c index d6060dc73..0bd33f517 100644 --- a/platform/linux-generic/pktio/null.c +++ b/platform/linux-generic/pktio/null.c @@ -157,7 +157,7 @@ static int null_outqueues_config(pktio_entry_t *pktio_entry ODP_UNUSED, static int null_init_global(void) { - ODP_PRINT("PKTIO: initialized null interface.\n"); + _ODP_PRINT("PKTIO: initialized null interface.\n"); return 0; } diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c index 768f876f5..20b6ec179 100644 --- a/platform/linux-generic/pktio/pcap.c +++ b/platform/linux-generic/pktio/pcap.c @@ -109,7 +109,7 @@ static int _pcapif_parse_devname(pkt_pcap_t *pcap, const char *devname) } else if (strncmp(tok, "loops=", 6) == 0) { pcap->loops = atoi(tok + 6); if (pcap->loops < 0) { - ODP_ERR("invalid loop count\n"); + _ODP_ERR("invalid loop count\n"); return -1; } } @@ -125,14 +125,13 @@ static int _pcapif_init_rx(pkt_pcap_t *pcap) pcap->rx = pcap_open_offline(pcap->fname_rx, errbuf); if (!pcap->rx) { - ODP_ERR("failed to open pcap file %s (%s)\n", - pcap->fname_rx, errbuf); + _ODP_ERR("failed to open pcap file %s (%s)\n", pcap->fname_rx, errbuf); return -1; } linktype = pcap_datalink(pcap->rx); if (linktype != DLT_EN10MB) { - ODP_ERR("unsupported datalink type: %d\n", linktype); + _ODP_ERR("unsupported datalink type: %d\n", linktype); return -1; } @@ -148,7 +147,7 @@ static int _pcapif_init_tx(pkt_pcap_t *pcap) * one needs to be opened for writing the dump */ tx = pcap_open_dead(DLT_EN10MB, PKTIO_PCAP_MTU_MAX); if (!tx) { - ODP_ERR("failed to open TX dump\n"); + _ODP_ERR("failed to open TX dump\n"); return -1; } @@ -157,8 +156,7 @@ static int _pcapif_init_tx(pkt_pcap_t *pcap) pcap->tx_dump = pcap_dump_open(tx, pcap->fname_tx); if (!pcap->tx_dump) { - ODP_ERR("failed to open dump file %s (%s)\n", - pcap->fname_tx, pcap_geterr(tx)); + _ODP_ERR("failed to open dump file %s (%s)\n", pcap->fname_tx, pcap_geterr(tx)); return -1; } @@ -192,14 +190,12 @@ static int pcapif_promisc_mode_set(pktio_entry_t *pktio_entry, if (pcap_compile(pcap->rx, &bpf, filter_exp, 0, PCAP_NETMASK_UNKNOWN) != 0) { - ODP_ERR("failed to compile promisc mode filter: %s\n", - pcap_geterr(pcap->rx)); + _ODP_ERR("failed to compile promisc mode filter: %s\n", pcap_geterr(pcap->rx)); return -1; } if (pcap_setfilter(pcap->rx, &bpf) != 0) { - ODP_ERR("failed to set promisc mode filter: %s\n", - pcap_geterr(pcap->rx)); + _ODP_ERR("failed to set promisc mode filter: %s\n", pcap_geterr(pcap->rx)); return -1; } @@ -274,8 +270,7 @@ static int _pcapif_reopen(pkt_pcap_t *pcap) pcap->rx = pcap_open_offline(pcap->fname_rx, errbuf); if (!pcap->rx) { - ODP_ERR("failed to reopen pcap file %s (%s)\n", - pcap->fname_rx, errbuf); + _ODP_ERR("failed to reopen pcap file %s (%s)\n", pcap->fname_rx, errbuf); return 1; } @@ -336,7 +331,7 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pull_head(pkt_hdr, frame_offset); if (odp_packet_copy_from_mem(pkt, 0, pkt_len, data) != 0) { - ODP_ERR("failed to copy packet data\n"); + _ODP_ERR("failed to copy packet data\n"); break; } @@ -530,7 +525,7 @@ static int pcapif_stats(pktio_entry_t *pktio_entry, static int pcapif_init_global(void) { - ODP_PRINT("PKTIO: initialized pcap interface.\n"); + _ODP_PRINT("PKTIO: initialized pcap interface.\n"); return 0; } diff --git a/platform/linux-generic/pktio/pktio_common.c b/platform/linux-generic/pktio/pktio_common.c index 5eaf40c33..097b8cf29 100644 --- a/platform/linux-generic/pktio/pktio_common.c +++ b/platform/linux-generic/pktio/pktio_common.c @@ -72,8 +72,8 @@ int _odp_sock_recv_mq_tmo_try_int_driven(const struct odp_pktin_queue_t queues[] entry[i] = get_pktio_entry(queues[i].pktio); index[i] = queues[i].index; if (entry[i] == NULL) { - ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", - (uintptr_t)queues[i].pktio); + _ODP_DBG("pktio entry %" PRIuPTR " does not exist\n", + (uintptr_t)queues[i].pktio); *trial_successful = 0; return -1; } diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c index 4ceb5151e..97574186c 100644 --- a/platform/linux-generic/pktio/socket.c +++ b/platform/linux-generic/pktio/socket.c @@ -97,7 +97,7 @@ static int sock_close(pktio_entry_t *pktio_entry) if (pkt_sock->sockfd != -1 && close(pkt_sock->sockfd) != 0) { _odp_errno = errno; - ODP_ERR("close(sockfd): %s\n", strerror(errno)); + _ODP_ERR("close(sockfd): %s\n", strerror(errno)); return -1; } @@ -129,7 +129,7 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, sockfd = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL)); if (sockfd == -1) { _odp_errno = errno; - ODP_ERR("socket(): %s\n", strerror(errno)); + _ODP_ERR("socket(): %s\n", strerror(errno)); goto error; } pkt_sock->sockfd = sockfd; @@ -140,8 +140,7 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, err = ioctl(sockfd, SIOCGIFINDEX, ðreq); if (err != 0) { _odp_errno = errno; - ODP_ERR("ioctl(SIOCGIFINDEX): %s: \"%s\".\n", strerror(errno), - ethreq.ifr_name); + _ODP_ERR("ioctl(SIOCGIFINDEX): %s: \"%s\".\n", strerror(errno), ethreq.ifr_name); goto error; } if_idx = ethreq.ifr_ifindex; @@ -164,14 +163,14 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, sa_ll.sll_protocol = htons(ETH_P_ALL); if (bind(sockfd, (struct sockaddr *)&sa_ll, sizeof(sa_ll)) < 0) { _odp_errno = errno; - ODP_ERR("bind(to IF): %s\n", strerror(errno)); + _ODP_ERR("bind(to IF): %s\n", strerror(errno)); goto error; } pktio_entry->stats_type = _odp_sock_stats_type_fd(pktio_entry, pkt_sock->sockfd); if (pktio_entry->stats_type == STATS_UNSUPPORTED) - ODP_DBG("pktio: %s unsupported stats\n", pktio_entry->name); + _ODP_DBG("pktio: %s unsupported stats\n", pktio_entry->name); err = sock_stats_reset(pktio_entry); if (err != 0) @@ -268,14 +267,14 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, if (odp_unlikely(msgvec[i].msg_hdr.msg_flags & MSG_TRUNC)) { odp_packet_free(pkt); - ODP_DBG("dropped truncated packet\n"); + _ODP_DBG("dropped truncated packet\n"); continue; } ret = odp_packet_trunc_tail(&pkt, odp_packet_len(pkt) - pkt_len, NULL, NULL); if (ret < 0) { - ODP_ERR("trunc_tail failed"); + _ODP_ERR("trunc_tail failed"); odp_packet_free(pkt); continue; } @@ -488,7 +487,7 @@ static int sock_mmsg_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, if (odp_unlikely(ret <= -1)) { if (i == 0 && SOCK_ERR_REPORT(errno)) { _odp_errno = errno; - ODP_ERR("sendmmsg(): %s\n", strerror(errno)); + _ODP_ERR("sendmmsg(): %s\n", strerror(errno)); odp_ticketlock_unlock(&pkt_sock->tx_lock); return -1; } @@ -625,11 +624,11 @@ static int sock_extra_stat_counter(pktio_entry_t *pktio_entry, uint32_t id, static int sock_init_global(void) { if (getenv("ODP_PKTIO_DISABLE_SOCKET_MMSG")) { - ODP_PRINT("PKTIO: socket mmsg skipped," + _ODP_PRINT("PKTIO: socket mmsg skipped," " enabled export ODP_PKTIO_DISABLE_SOCKET_MMSG=1.\n"); disable_pktio = 1; } else { - ODP_PRINT("PKTIO: initialized socket mmsg," + _ODP_PRINT("PKTIO: initialized socket mmsg," " use export ODP_PKTIO_DISABLE_SOCKET_MMSG=1 to disable.\n"); } return 0; diff --git a/platform/linux-generic/pktio/socket_common.c b/platform/linux-generic/pktio/socket_common.c index ab0caeb81..b6ae7b9ae 100644 --- a/platform/linux-generic/pktio/socket_common.c +++ b/platform/linux-generic/pktio/socket_common.c @@ -58,8 +58,7 @@ int _odp_mac_addr_get_fd(int fd, const char *name, unsigned char mac_dst[]) ret = ioctl(fd, SIOCGIFHWADDR, ðreq); if (ret != 0) { _odp_errno = errno; - ODP_ERR("ioctl(SIOCGIFHWADDR): %s: \"%s\".\n", strerror(errno), - ethreq.ifr_name); + _ODP_ERR("ioctl(SIOCGIFHWADDR): %s: \"%s\".\n", strerror(errno), ethreq.ifr_name); return -1; } @@ -83,8 +82,7 @@ uint32_t _odp_mtu_get_fd(int fd, const char *name) ret = ioctl(fd, SIOCGIFMTU, &ifr); if (ret < 0) { _odp_errno = errno; - ODP_ERR("ioctl(SIOCGIFMTU): %s: \"%s\".\n", strerror(errno), - ifr.ifr_name); + _ODP_ERR("ioctl(SIOCGIFMTU): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return 0; } return ifr.ifr_mtu + _ODP_ETHHDR_LEN; @@ -107,8 +105,7 @@ int _odp_mtu_set_fd(int fd, const char *name, int mtu) ret = ioctl(fd, SIOCSIFMTU, &ifr); if (ret < 0) { _odp_errno = errno; - ODP_ERR("ioctl(SIOCSIFMTU): %s: \"%s\".\n", strerror(errno), - ifr.ifr_name); + _ODP_ERR("ioctl(SIOCSIFMTU): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; } return 0; @@ -128,8 +125,7 @@ int _odp_promisc_mode_set_fd(int fd, const char *name, int enable) ret = ioctl(fd, SIOCGIFFLAGS, &ifr); if (ret < 0) { _odp_errno = errno; - ODP_DBG("ioctl(SIOCGIFFLAGS): %s: \"%s\".\n", strerror(errno), - ifr.ifr_name); + _ODP_DBG("ioctl(SIOCGIFFLAGS): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; } @@ -141,8 +137,7 @@ int _odp_promisc_mode_set_fd(int fd, const char *name, int enable) ret = ioctl(fd, SIOCSIFFLAGS, &ifr); if (ret < 0) { _odp_errno = errno; - ODP_DBG("ioctl(SIOCSIFFLAGS): %s: \"%s\".\n", strerror(errno), - ifr.ifr_name); + _ODP_DBG("ioctl(SIOCSIFFLAGS): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; } return 0; @@ -162,8 +157,7 @@ int _odp_promisc_mode_get_fd(int fd, const char *name) ret = ioctl(fd, SIOCGIFFLAGS, &ifr); if (ret < 0) { _odp_errno = errno; - ODP_DBG("ioctl(SIOCGIFFLAGS): %s: \"%s\".\n", strerror(errno), - ifr.ifr_name); + _ODP_DBG("ioctl(SIOCGIFFLAGS): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; } @@ -179,8 +173,7 @@ int _odp_link_status_fd(int fd, const char *name) ret = ioctl(fd, SIOCGIFFLAGS, &ifr); if (ret < 0) { _odp_errno = errno; - ODP_DBG("ioctl(SIOCGIFFLAGS): %s: \"%s\".\n", strerror(errno), - ifr.ifr_name); + _ODP_DBG("ioctl(SIOCGIFFLAGS): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return ODP_PKTIO_LINK_STATUS_UNKNOWN; } @@ -207,8 +200,7 @@ int _odp_link_info_fd(int fd, const char *name, odp_pktio_link_info_t *info) ifr.ifr_data = (void *)&pcmd; if (ioctl(fd, SIOCETHTOOL, &ifr) && errno != EOPNOTSUPP) { _odp_errno = errno; - ODP_ERR("ioctl(SIOCETHTOOL): %s: \"%s\".\n", strerror(errno), - ifr.ifr_name); + _ODP_ERR("ioctl(SIOCETHTOOL): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; } @@ -220,7 +212,8 @@ int _odp_link_info_fd(int fd, const char *name, odp_pktio_link_info_t *info) ifr.ifr_data = (void *)&ecmd_old; if (ioctl(fd, SIOCETHTOOL, &ifr) < 0) { _odp_errno = errno; - ODP_ERR("ioctl(SIOCETHTOOL): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); + _ODP_ERR("ioctl(SIOCETHTOOL): %s: \"%s\".\n", strerror(errno), + ifr.ifr_name); return -1; } @@ -261,7 +254,7 @@ int _odp_link_info_fd(int fd, const char *name, odp_pktio_link_info_t *info) } if (hcmd.link_mode_masks_nwords >= 0 || hcmd.cmd != ETHTOOL_GLINKSETTINGS) { - ODP_ERR("ETHTOOL_GLINKSETTINGS handshake failed\n"); + _ODP_ERR("ETHTOOL_GLINKSETTINGS handshake failed\n"); return -1; } /* Absolute value indicates kernel recommended 'link_mode_masks_nwords' value. */ @@ -278,8 +271,7 @@ int _odp_link_info_fd(int fd, const char *name, odp_pktio_link_info_t *info) ifr.ifr_data = (void *)ecmd; if (ioctl(fd, SIOCETHTOOL, &ifr) < 0) { _odp_errno = errno; - ODP_ERR("ioctl(SIOCETHTOOL): %s: \"%s\".\n", strerror(errno), - ifr.ifr_name); + _ODP_ERR("ioctl(SIOCETHTOOL): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; } diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 4e1bf8527..ea6c0b9b7 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -114,14 +114,14 @@ static int mmap_pkt_socket(void) if (sock == -1) { _odp_errno = errno; - ODP_ERR("socket(SOCK_RAW): %s\n", strerror(errno)); + _ODP_ERR("socket(SOCK_RAW): %s\n", strerror(errno)); return -1; } ret = setsockopt(sock, SOL_PACKET, PACKET_VERSION, &ver, sizeof(ver)); if (ret == -1) { _odp_errno = errno; - ODP_ERR("setsockopt(PACKET_VERSION): %s\n", strerror(errno)); + _ODP_ERR("setsockopt(PACKET_VERSION): %s\n", strerror(errno)); close(sock); return -1; } @@ -187,7 +187,7 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, if (odp_unlikely(pkt_len > pkt_sock->mtu)) { tp_hdr->tp_status = TP_STATUS_KERNEL; frame_num = next_frame_num; - ODP_DBG("dropped oversized packet\n"); + _ODP_DBG("dropped oversized packet\n"); continue; } @@ -251,7 +251,7 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, static int warning_printed; if (warning_printed == 0) { - ODP_DBG("Original TPID value lost. Using 0x8100 for single tagged and 0x88a8 for double tagged.\n"); + _ODP_DBG("Original TPID value lost. Using 0x8100 for single tagged and 0x88a8 for double tagged.\n"); warning_printed = 1; } type2 = (uint16_t *)(uintptr_t)(mac + (2 * _ODP_ETHADDR_LEN) + vlan_len); @@ -347,7 +347,7 @@ static inline int pkt_mmap_v2_tx(pktio_entry_t *pktio_entry, int sock, if (tp_status != TP_STATUS_AVAILABLE) { if (tp_status == TP_STATUS_WRONG_FORMAT) { - ODP_ERR("Socket mmap: wrong format\n"); + _ODP_ERR("Socket mmap: wrong format\n"); return -1; } @@ -389,8 +389,7 @@ static inline int pkt_mmap_v2_tx(pktio_entry_t *pktio_entry, int sock, /* Returns -1 when nothing is sent (send() would block) */ if (ret < 0 && errno != EWOULDBLOCK) { - ODP_ERR("Socket mmap: send failed, ret %i, errno %i\n", - ret, errno); + _ODP_ERR("Socket mmap: send failed, ret %i, errno %i\n", ret, errno); return -1; } @@ -404,7 +403,7 @@ static inline int pkt_mmap_v2_tx(pktio_entry_t *pktio_entry, int sock, break; if (tp_status == TP_STATUS_WRONG_FORMAT) { - ODP_ERR("Socket mmap: wrong format\n"); + _ODP_ERR("Socket mmap: wrong format\n"); break; } } @@ -462,7 +461,7 @@ static int mmap_setup_ring(pkt_sock_mmap_t *pkt_sock, struct ring *ring, shm = odp_shm_reserve(NULL, ring_size, ODP_CACHE_LINE_SIZE, flags); if (shm == ODP_SHM_INVALID) { - ODP_ERR("Reserving shm failed\n"); + _ODP_ERR("Reserving shm failed\n"); return -1; } ring->shm = shm; @@ -477,21 +476,21 @@ static int mmap_setup_ring(pkt_sock_mmap_t *pkt_sock, struct ring *ring, ring->flen = ring->req.tp_frame_size; ring->rd_len = ring_size; - ODP_DBG(" tp_block_size %u\n", ring->req.tp_block_size); - ODP_DBG(" tp_block_nr %u\n", ring->req.tp_block_nr); - ODP_DBG(" tp_frame_size %u\n", ring->req.tp_frame_size); - ODP_DBG(" tp_frame_nr %u\n", ring->req.tp_frame_nr); + _ODP_DBG(" tp_block_size %u\n", ring->req.tp_block_size); + _ODP_DBG(" tp_block_nr %u\n", ring->req.tp_block_nr); + _ODP_DBG(" tp_frame_size %u\n", ring->req.tp_frame_size); + _ODP_DBG(" tp_frame_nr %u\n", ring->req.tp_frame_nr); ret = setsockopt(sock, SOL_PACKET, type, &ring->req, sizeof(ring->req)); if (ret == -1) { _odp_errno = errno; - ODP_ERR("setsockopt(pkt mmap): %s\n", strerror(errno)); + _ODP_ERR("setsockopt(pkt mmap): %s\n", strerror(errno)); return -1; } ring->rd = odp_shm_addr(shm); if (!ring->rd) { - ODP_ERR("Reading shm addr failed\n"); + _ODP_ERR("Reading shm addr failed\n"); return -1; } @@ -516,7 +515,7 @@ static int mmap_sock(pkt_sock_mmap_t *pkt_sock) if (pkt_sock->mmap_base == MAP_FAILED) { _odp_errno = errno; - ODP_ERR("mmap rx&tx buffer failed: %s\n", strerror(errno)); + _ODP_ERR("mmap rx&tx buffer failed: %s\n", strerror(errno)); return -1; } @@ -572,7 +571,7 @@ static int mmap_bind_sock(pkt_sock_mmap_t *pkt_sock, const char *netdev) sizeof(pkt_sock->ll)); if (ret == -1) { _odp_errno = errno; - ODP_ERR("bind(to IF): %s\n", strerror(errno)); + _ODP_ERR("bind(to IF): %s\n", strerror(errno)); return -1; } @@ -586,13 +585,13 @@ static int sock_mmap_close(pktio_entry_t *entry) ret = mmap_unmap_sock(pkt_sock); if (ret != 0) { - ODP_ERR("mmap_unmap_sock() %s\n", strerror(errno)); + _ODP_ERR("mmap_unmap_sock() %s\n", strerror(errno)); return -1; } if (pkt_sock->sockfd != -1 && close(pkt_sock->sockfd) != 0) { _odp_errno = errno; - ODP_ERR("close(sockfd): %s\n", strerror(errno)); + _ODP_ERR("close(sockfd): %s\n", strerror(errno)); return -1; } @@ -643,14 +642,14 @@ static int sock_mmap_open(odp_pktio_t id ODP_UNUSED, if (pkt_sock->mtu > _ODP_SOCKET_MTU_MAX) pkt_sock->mtu_max = pkt_sock->mtu; - ODP_DBG("MTU size: %i\n", pkt_sock->mtu); + _ODP_DBG("MTU size: %i\n", pkt_sock->mtu); - ODP_DBG("TX ring setup:\n"); + _ODP_DBG("TX ring setup:\n"); ret = mmap_setup_ring(pkt_sock, &pkt_sock->tx_ring, PACKET_TX_RING); if (ret != 0) goto error; - ODP_DBG("RX ring setup:\n"); + _ODP_DBG("RX ring setup:\n"); ret = mmap_setup_ring(pkt_sock, &pkt_sock->rx_ring, PACKET_RX_RING); if (ret != 0) goto error; @@ -666,14 +665,14 @@ static int sock_mmap_open(odp_pktio_t id ODP_UNUSED, if_idx = if_nametoindex(netdev); if (if_idx == 0) { _odp_errno = errno; - ODP_ERR("if_nametoindex(): %s\n", strerror(errno)); + _ODP_ERR("if_nametoindex(): %s\n", strerror(errno)); goto error; } pktio_entry->stats_type = _odp_sock_stats_type_fd(pktio_entry, pkt_sock->sockfd); if (pktio_entry->stats_type == STATS_UNSUPPORTED) - ODP_DBG("pktio: %s unsupported stats\n", pktio_entry->name); + _ODP_DBG("pktio: %s unsupported stats\n", pktio_entry->name); ret = _odp_sock_stats_reset_fd(pktio_entry, pkt_priv(pktio_entry)->sockfd); @@ -934,11 +933,11 @@ static int sock_mmap_extra_stat_counter(pktio_entry_t *pktio_entry, uint32_t id, static int sock_mmap_init_global(void) { if (getenv("ODP_PKTIO_DISABLE_SOCKET_MMAP")) { - ODP_PRINT("PKTIO: socket mmap skipped," + _ODP_PRINT("PKTIO: socket mmap skipped," " enabled export ODP_PKTIO_DISABLE_SOCKET_MMAP=1.\n"); disable_pktio = 1; } else { - ODP_PRINT("PKTIO: initialized socket mmap," + _ODP_PRINT("PKTIO: initialized socket mmap," " use export ODP_PKTIO_DISABLE_SOCKET_MMAP=1 to disable.\n"); } return 0; diff --git a/platform/linux-generic/pktio/socket_xdp.c b/platform/linux-generic/pktio/socket_xdp.c index 2b4abf66c..b10fc9fce 100644 --- a/platform/linux-generic/pktio/socket_xdp.c +++ b/platform/linux-generic/pktio/socket_xdp.c @@ -121,11 +121,11 @@ static odp_bool_t disable_pktio; static int sock_xdp_init_global(void) { if (getenv("ODP_PKTIO_DISABLE_SOCKET_XDP")) { - ODP_PRINT("PKTIO: socket xdp skipped," + _ODP_PRINT("PKTIO: socket xdp skipped," " enabled export ODP_PKTIO_DISABLE_SOCKET_XDP=1.\n"); disable_pktio = true; } else { - ODP_PRINT("PKTIO: initialized socket xdp," + _ODP_PRINT("PKTIO: initialized socket xdp," " use export ODP_PKTIO_DISABLE_SOCKET_XDP=1 to disable.\n"); } @@ -143,16 +143,16 @@ static void parse_options(xdp_umem_info_t *umem_info) &umem_info->num_rx_desc) || !_odp_libconfig_lookup_ext_int(CONF_BASE_STR, NULL, TX_DESCS_STR, &umem_info->num_tx_desc)) { - ODP_ERR("Unable to parse xdp descriptor configuration, using defaults (%d).\n", - NUM_DESCS_DEFAULT); + _ODP_ERR("Unable to parse xdp descriptor configuration, using defaults (%d).\n", + NUM_DESCS_DEFAULT); goto defaults; } if (umem_info->num_rx_desc <= 0 || umem_info->num_tx_desc <= 0 || !_ODP_CHECK_IS_POWER2(umem_info->num_rx_desc) || !_ODP_CHECK_IS_POWER2(umem_info->num_tx_desc)) { - ODP_ERR("Invalid xdp descriptor configuration, using defaults (%d).\n", - NUM_DESCS_DEFAULT); + _ODP_ERR("Invalid xdp descriptor configuration, using defaults (%d).\n", + NUM_DESCS_DEFAULT); goto defaults; } @@ -251,7 +251,7 @@ static int sock_xdp_open(odp_pktio_t pktio, pktio_entry_t *pktio_entry, const ch ret = umem_create(priv->umem_info, pool); if (ret) { - ODP_ERR("Error creating UMEM pool for xdp: %d\n", ret); + _ODP_ERR("Error creating UMEM pool for xdp: %d\n", ret); return -1; } @@ -263,7 +263,7 @@ static int sock_xdp_open(odp_pktio_t pktio, pktio_entry_t *pktio_entry, const ch ret = socket(AF_INET, SOCK_DGRAM, 0); if (ret == -1) { - ODP_ERR("Error creating helper socket for xdp: %s\n", strerror(errno)); + _ODP_ERR("Error creating helper socket for xdp: %s\n", strerror(errno)); goto sock_err; } @@ -282,10 +282,10 @@ static int sock_xdp_open(odp_pktio_t pktio, pktio_entry_t *pktio_entry, const ch priv->bind_q = get_bind_queue_index(pktio_entry->name); - ODP_DBG("Socket xdp interface (%s):\n", pktio_entry->name); - ODP_DBG(" num_rx_desc: %d\n", priv->umem_info->num_rx_desc); - ODP_DBG(" num_tx_desc: %d\n", priv->umem_info->num_tx_desc); - ODP_DBG(" starting bind queue: %u\n", priv->bind_q); + _ODP_DBG("Socket xdp interface (%s):\n", pktio_entry->name); + _ODP_DBG(" num_rx_desc: %d\n", priv->umem_info->num_rx_desc); + _ODP_DBG(" num_tx_desc: %d\n", priv->umem_info->num_tx_desc); + _ODP_DBG(" starting bind queue: %u\n", priv->bind_q); return 0; @@ -465,7 +465,7 @@ static int sock_xdp_extra_stat_counter(pktio_entry_t *pktio_entry, uint32_t id, const uint32_t total_stats = MAX_INTERNAL_STATS * priv->num_q; if (id >= total_stats) { - ODP_ERR("Invalid counter id: %u (allowed range: 0-%u)\n", id, total_stats - 1U); + _ODP_ERR("Invalid counter id: %u (allowed range: 0-%u)\n", id, total_stats - 1U); return -1; } @@ -652,9 +652,9 @@ static void handle_pending_tx(xdp_sock_t *sock, uint8_t *base_addr, int num) compl_q = &sock->compl_q; sent = xsk_ring_cons__peek(compl_q, num, &start_idx); - odp_packet_t packets[sent]; - if (sent) { + odp_packet_t packets[sent]; + for (uint32_t i = 0U; i < sent; ++i) { frame_off = *xsk_ring_cons__comp_addr(compl_q, start_idx++); frame_off = xsk_umem__extract_addr(frame_off); @@ -845,7 +845,7 @@ static int set_queue_capability(int fd, const char *devname, odp_pktio_capabilit if (ret == -1 || channels.max_combined == 0U) { if (ret == -1 && errno != EOPNOTSUPP) { - ODP_ERR("Unable to query NIC channel capabilities: %s\n", strerror(errno)); + _ODP_ERR("Unable to query NIC channel capabilities: %s\n", strerror(errno)); return -1; } @@ -940,15 +940,15 @@ static int sock_xdp_output_queues_config(pktio_entry_t *pktio_entry, &sock->tx, &sock->fill_q, &sock->compl_q, &config); if (ret) { - ODP_ERR("Error creating xdp socket for bind queue %u: %d\n", bind_q, ret); + _ODP_ERR("Error creating xdp socket for bind queue %u: %d\n", bind_q, ret); goto err; } ++i; if (!reserve_fill_queue_elements(priv, sock, config.rx_size)) { - ODP_ERR("Unable to reserve fill queue descriptors for queue: %u.\n", - bind_q); + _ODP_ERR("Unable to reserve fill queue descriptors for queue: %u.\n", + bind_q); goto err; } @@ -1032,8 +1032,8 @@ static void sock_xdp_adjust_block_size(uint8_t *data ODP_UNUSED, uint32_t *block return; if (size > ps) { - ODP_ERR("Adjusted pool block size larger than page size: %u > %" PRIu64 "\n", - size, ps); + _ODP_ERR("Adjusted pool block size larger than page size: %u > %" PRIu64 "\n", + size, ps); *block_size = 0U; } diff --git a/platform/linux-generic/pktio/stats/ethtool_stats.c b/platform/linux-generic/pktio/stats/ethtool_stats.c index f300ed0ba..678ec45ee 100644 --- a/platform/linux-generic/pktio/stats/ethtool_stats.c +++ b/platform/linux-generic/pktio/stats/ethtool_stats.c @@ -51,7 +51,7 @@ static struct ethtool_gstrings *get_stringset(int fd, struct ifreq *ifr) ifr->ifr_data = (void *)&drvinfo; if (ioctl(fd, SIOCETHTOOL, ifr)) { _odp_errno = errno; - ODP_ERR("Cannot get stats information\n"); + _ODP_ERR("Cannot get stats information\n"); return NULL; } len = *(uint32_t *)(void *)((char *)&drvinfo + drvinfo_offset); @@ -61,13 +61,13 @@ static struct ethtool_gstrings *get_stringset(int fd, struct ifreq *ifr) } if (!len) { - ODP_ERR("len is zero"); + _ODP_ERR("len is zero"); return NULL; } strings = calloc(1, sizeof(*strings) + len * ETH_GSTRING_LEN); if (!strings) { - ODP_ERR("alloc failed\n"); + _ODP_ERR("alloc failed\n"); return NULL; } @@ -77,7 +77,7 @@ static struct ethtool_gstrings *get_stringset(int fd, struct ifreq *ifr) ifr->ifr_data = (void *)strings; if (ioctl(fd, SIOCETHTOOL, ifr)) { _odp_errno = errno; - ODP_ERR("Cannot get stats information\n"); + _ODP_ERR("Cannot get stats information\n"); free(strings); return NULL; } @@ -105,7 +105,7 @@ static int ethtool_stats_get(int fd, const char *name, n_stats = strings->len; if (n_stats < 1) { - ODP_ERR("no stats available\n"); + _ODP_ERR("no stats available\n"); free(strings); return -1; } @@ -272,7 +272,7 @@ int _odp_ethtool_extra_stat_counter(int fd, const char *name, uint32_t id, return -1; if (id >= n_stats) { - ODP_ERR("Invalid counter id\n"); + _ODP_ERR("Invalid counter id\n"); ret = -1; } else { *stat = estats->data[id]; diff --git a/platform/linux-generic/pktio/stats/packet_io_stats.c b/platform/linux-generic/pktio/stats/packet_io_stats.c index 54df874f0..ac61c0343 100644 --- a/platform/linux-generic/pktio/stats/packet_io_stats.c +++ b/platform/linux-generic/pktio/stats/packet_io_stats.c @@ -24,7 +24,7 @@ static int sock_stats_get(pktio_entry_t *e, odp_pktio_stats_t *stats, int fd) ret = _odp_sysfs_stats(e, stats); if (ret) - ODP_ERR("Failed to get pktio statistics.\n"); + _ODP_ERR("Failed to get pktio statistics.\n"); return ret; } diff --git a/platform/linux-generic/pktio/stats/sysfs_stats.c b/platform/linux-generic/pktio/stats/sysfs_stats.c index 94872dfd7..d9fb0754c 100644 --- a/platform/linux-generic/pktio/stats/sysfs_stats.c +++ b/platform/linux-generic/pktio/stats/sysfs_stats.c @@ -32,7 +32,7 @@ static int sysfs_get_val(const char *fname, uint64_t *val) * kernel driver. */ if (errno != ENOENT) - ODP_ERR("fopen %s: %s\n", fname, strerror(errno)); + _ODP_ERR("fopen %s: %s\n", fname, strerror(errno)); return 0; } @@ -42,7 +42,7 @@ static int sysfs_get_val(const char *fname, uint64_t *val) (void)fclose(file); if (ret != 1) { - ODP_ERR("read %s\n", fname); + _ODP_ERR("read %s\n", fname); return -1; } @@ -103,7 +103,7 @@ int _odp_sysfs_extra_stat_info(pktio_entry_t *pktio_entry, snprintf(sysfs_dir, PATH_MAX, SYSFS_DIR, pktio_entry->name); dir = opendir(sysfs_dir); if (!dir) { - ODP_ERR("Failed to open sysfs dir: %s\n", sysfs_dir); + _ODP_ERR("Failed to open sysfs dir: %s\n", sysfs_dir); return -1; } @@ -135,7 +135,7 @@ int _odp_sysfs_extra_stats(pktio_entry_t *pktio_entry, uint64_t stats[], snprintf(sysfs_dir, PATH_MAX, SYSFS_DIR, pktio_entry->name); dir = opendir(sysfs_dir); if (!dir) { - ODP_ERR("Failed to open dir: %s\n", sysfs_dir); + _ODP_ERR("Failed to open dir: %s\n", sysfs_dir); return -1; } @@ -148,7 +148,7 @@ int _odp_sysfs_extra_stats(pktio_entry_t *pktio_entry, uint64_t stats[], snprintf(file_path, PATH_MAX, "%s/%s", sysfs_dir, e->d_name); if (sysfs_get_val(file_path, &val)) { - ODP_ERR("Failed to read file: %s/n", file_path); + _ODP_ERR("Failed to read file: %s/n", file_path); counters = -1; break; } @@ -176,7 +176,7 @@ int _odp_sysfs_extra_stat_counter(pktio_entry_t *pktio_entry, uint32_t id, snprintf(sysfs_dir, PATH_MAX, SYSFS_DIR, pktio_entry->name); dir = opendir(sysfs_dir); if (!dir) { - ODP_ERR("Failed to open dir: %s\n", sysfs_dir); + _ODP_ERR("Failed to open dir: %s\n", sysfs_dir); return -1; } @@ -191,7 +191,7 @@ int _odp_sysfs_extra_stat_counter(pktio_entry_t *pktio_entry, uint32_t id, snprintf(file_path, PATH_MAX, "%s/%s", sysfs_dir, e->d_name); if (sysfs_get_val(file_path, &val)) { - ODP_ERR("Failed to read file: %s/n", file_path); + _ODP_ERR("Failed to read file: %s/n", file_path); } else { *stat = val; ret = 0; diff --git a/platform/linux-generic/pktio/tap.c b/platform/linux-generic/pktio/tap.c index 7b0df4cac..604e53170 100644 --- a/platform/linux-generic/pktio/tap.c +++ b/platform/linux-generic/pktio/tap.c @@ -78,7 +78,7 @@ static int gen_random_mac(unsigned char *mac) { mac[0] = 0x7a; /* not multicast and local assignment bit is set */ if (odp_random_data(mac + 1, 5, ODP_RANDOM_BASIC) < 5) { - ODP_ERR("odp_random_data failed.\n"); + _ODP_ERR("odp_random_data failed.\n"); return -1; } return 0; @@ -99,8 +99,7 @@ static int mac_addr_set_fd(int fd, const char *name, ret = ioctl(fd, SIOCSIFHWADDR, ðreq); if (ret != 0) { _odp_errno = errno; - ODP_ERR("ioctl(SIOCSIFHWADDR): %s: \"%s\".\n", strerror(errno), - ethreq.ifr_name); + _ODP_ERR("ioctl(SIOCSIFHWADDR): %s: \"%s\".\n", strerror(errno), ethreq.ifr_name); return -1; } @@ -130,7 +129,7 @@ static int tap_pktio_open(odp_pktio_t id ODP_UNUSED, fd = open("/dev/net/tun", O_RDWR); if (fd < 0) { _odp_errno = errno; - ODP_ERR("failed to open /dev/net/tun: %s\n", strerror(errno)); + _ODP_ERR("failed to open /dev/net/tun: %s\n", strerror(errno)); return -1; } @@ -145,8 +144,7 @@ static int tap_pktio_open(odp_pktio_t id ODP_UNUSED, if (ioctl(fd, TUNSETIFF, (void *)&ifr) < 0) { _odp_errno = errno; - ODP_ERR("%s: creating tap device failed: %s\n", - ifr.ifr_name, strerror(errno)); + _ODP_ERR("%s: creating tap device failed: %s\n", ifr.ifr_name, strerror(errno)); goto tap_err; } @@ -154,13 +152,13 @@ static int tap_pktio_open(odp_pktio_t id ODP_UNUSED, flags = fcntl(fd, F_GETFL, 0); if (flags < 0) { _odp_errno = errno; - ODP_ERR("fcntl(F_GETFL) failed: %s\n", strerror(errno)); + _ODP_ERR("fcntl(F_GETFL) failed: %s\n", strerror(errno)); goto tap_err; } if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) { _odp_errno = errno; - ODP_ERR("fcntl(F_SETFL) failed: %s\n", strerror(errno)); + _ODP_ERR("fcntl(F_SETFL) failed: %s\n", strerror(errno)); goto tap_err; } @@ -171,14 +169,14 @@ static int tap_pktio_open(odp_pktio_t id ODP_UNUSED, skfd = socket(AF_INET, SOCK_DGRAM, 0); if (skfd < 0) { _odp_errno = errno; - ODP_ERR("socket creation failed: %s\n", strerror(errno)); + _ODP_ERR("socket creation failed: %s\n", strerror(errno)); goto tap_err; } mtu = _odp_mtu_get_fd(skfd, devname + 4); if (mtu == 0) { _odp_errno = errno; - ODP_ERR("_odp_mtu_get_fd failed: %s\n", strerror(errno)); + _ODP_ERR("_odp_mtu_get_fd failed: %s\n", strerror(errno)); goto sock_err; } tap->mtu_max = _ODP_SOCKET_MTU_MAX; @@ -194,7 +192,7 @@ sock_err: close(skfd); tap_err: close(fd); - ODP_ERR("Tap device alloc failed.\n"); + _ODP_ERR("Tap device alloc failed.\n"); return -1; } @@ -210,7 +208,7 @@ static int tap_pktio_start(pktio_entry_t *pktio_entry) /* Up interface by default. */ if (ioctl(tap->skfd, SIOCGIFFLAGS, &ifr) < 0) { _odp_errno = errno; - ODP_ERR("ioctl(SIOCGIFFLAGS) failed: %s\n", strerror(errno)); + _ODP_ERR("ioctl(SIOCGIFFLAGS) failed: %s\n", strerror(errno)); goto sock_err; } @@ -219,13 +217,13 @@ static int tap_pktio_start(pktio_entry_t *pktio_entry) if (ioctl(tap->skfd, SIOCSIFFLAGS, &ifr) < 0) { _odp_errno = errno; - ODP_ERR("failed to come up: %s\n", strerror(errno)); + _ODP_ERR("failed to come up: %s\n", strerror(errno)); goto sock_err; } return 0; sock_err: - ODP_ERR("Tap device open failed.\n"); + _ODP_ERR("Tap device open failed.\n"); return -1; } @@ -241,7 +239,7 @@ static int tap_pktio_stop(pktio_entry_t *pktio_entry) /* Up interface by default. */ if (ioctl(tap->skfd, SIOCGIFFLAGS, &ifr) < 0) { _odp_errno = errno; - ODP_ERR("ioctl(SIOCGIFFLAGS) failed: %s\n", strerror(errno)); + _ODP_ERR("ioctl(SIOCGIFFLAGS) failed: %s\n", strerror(errno)); goto sock_err; } @@ -250,13 +248,13 @@ static int tap_pktio_stop(pktio_entry_t *pktio_entry) if (ioctl(tap->skfd, SIOCSIFFLAGS, &ifr) < 0) { _odp_errno = errno; - ODP_ERR("failed to come up: %s\n", strerror(errno)); + _ODP_ERR("failed to come up: %s\n", strerror(errno)); goto sock_err; } return 0; sock_err: - ODP_ERR("Tap device open failed.\n"); + _ODP_ERR("Tap device open failed.\n"); return -1; } @@ -267,13 +265,13 @@ static int tap_pktio_close(pktio_entry_t *pktio_entry) if (tap->fd != -1 && close(tap->fd) != 0) { _odp_errno = errno; - ODP_ERR("close(tap->fd): %s\n", strerror(errno)); + _ODP_ERR("close(tap->fd): %s\n", strerror(errno)); ret = -1; } if (tap->skfd != -1 && close(tap->skfd) != 0) { _odp_errno = errno; - ODP_ERR("close(tap->skfd): %s\n", strerror(errno)); + _ODP_ERR("close(tap->skfd): %s\n", strerror(errno)); ret = -1; } @@ -301,7 +299,7 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data, pull_head(pkt_hdr, frame_offset); if (odp_packet_copy_from_mem(pkt, 0, len, data) < 0) { - ODP_ERR("failed to copy packet data\n"); + _ODP_ERR("failed to copy packet data\n"); odp_packet_free(pkt); return ODP_PACKET_INVALID; } @@ -401,7 +399,7 @@ static int tap_pktio_send_lockless(pktio_entry_t *pktio_entry, } if (odp_packet_copy_to_mem(pkts[i], 0, pkt_len, buf) < 0) { - ODP_ERR("failed to copy packet data\n"); + _ODP_ERR("failed to copy packet data\n"); break; } @@ -412,12 +410,12 @@ static int tap_pktio_send_lockless(pktio_entry_t *pktio_entry, if (retval < 0) { if (i == 0 && SOCK_ERR_REPORT(errno)) { _odp_errno = errno; - ODP_ERR("write(): %s\n", strerror(errno)); + _ODP_ERR("write(): %s\n", strerror(errno)); return -1; } break; } else if ((uint32_t)retval != pkt_len) { - ODP_ERR("sent partial ethernet packet\n"); + _ODP_ERR("sent partial ethernet packet\n"); if (i == 0) { _odp_errno = EMSGSIZE; return -1; diff --git a/platform/linux-generic/test/Makefile.am b/platform/linux-generic/test/Makefile.am index ca1794b63..06e5a3ba2 100644 --- a/platform/linux-generic/test/Makefile.am +++ b/platform/linux-generic/test/Makefile.am @@ -52,8 +52,3 @@ $(TESTENV): echo "TESTS=\"$(TESTS)\"" > $@ echo "$(TESTS_ENVIRONMENT)" >> $@ echo "$(LOG_COMPILER)" >> $@ - -if test_installdir -installcheck-local: - $(DESTDIR)/$(testdir)/run-test.sh $(TESTNAME) -endif diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index ce132faeb..2f333bccb 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -14,42 +14,10 @@ make -j $(nproc) make install -pushd ${HOME} - -# Fix build on CentOS -PKG_CONFIG="${TARGET_ARCH}-pkg-config" -if ! [ -x "$(command -v ${PKG_CONFIG})" ]; then - PKG_CONFIG="pkg-config" -fi - -# Default ODP library name -if [ -z "$ODP_LIB_NAME" ] ; then -ODP_LIB_NAME=libodp-dpdk -fi - -# Additional warning checks -EXTRA_CHECKS="-Werror -Wall -Wextra -Wfloat-equal -Wpacked" -# Ignore clang warning about large atomic operations causing significant performance penalty -if [ "${CC#clang}" != "${CC}" ] ; then - EXTRA_CHECKS="${EXTRA_CHECKS} -Wno-unknown-warning-option -Wno-atomic-alignment" -fi -# Ignore warnings from aarch64 DPDK internals -if [ "${TARGET_ARCH}" == "aarch64-linux-gnu" ] ; then - EXTRA_CHECKS="${EXTRA_CHECKS} -Wno-packed" -fi - -${CC} ${CFLAGS} ${EXTRA_CHECKS} ${OLDPWD}/example/sysinfo/odp_sysinfo.c -o odp_sysinfo_inst_dynamic \ - `PKG_CONFIG_PATH=/opt/odp/lib/pkgconfig:${PKG_CONFIG_PATH} ${PKG_CONFIG} --cflags --libs ${ODP_LIB_NAME}` - -sysctl vm.nr_hugepages=1000 +echo 1000 | tee /proc/sys/vm/nr_hugepages mkdir -p /mnt/huge mount -t hugetlbfs nodev /mnt/huge -if [ -z "$TARGET_ARCH" ] || [ "$TARGET_ARCH" == "$BUILD_ARCH" ] -then - LD_LIBRARY_PATH="/opt/odp/lib:$LD_LIBRARY_PATH" ./odp_sysinfo_inst_dynamic -fi -popd +make installcheck -#dpdk wrapper script can umount hugepages itself -umount /mnt/huge || true +umount /mnt/huge diff --git a/scripts/ci/build_ppc64el.sh b/scripts/ci/build_ppc64el.sh index 984481bb5..93dd36290 100755 --- a/scripts/ci/build_ppc64el.sh +++ b/scripts/ci/build_ppc64el.sh @@ -7,6 +7,10 @@ if [ "${CC#clang}" != "${CC}" ] ; then export CXX="clang++ --target=${TARGET_ARCH}" # DPDK clang build broken export CONF="${CONF} --disable-dpdk" + + # Ignore warnings about large atomic operation performance penalty + export ODP_CFLAGS_EXTRA=-Wno-error=atomic-alignment + else export CC="${TARGET_ARCH}-gcc" export CXX="${TARGET_ARCH}-g++" diff --git a/scripts/ci/build_riscv64.sh b/scripts/ci/build_riscv64.sh index bf6fdf941..c395aac82 100755 --- a/scripts/ci/build_riscv64.sh +++ b/scripts/ci/build_riscv64.sh @@ -27,6 +27,4 @@ make -j $(nproc) make install -pushd ${HOME} -${CC} ${CFLAGS} ${OLDPWD}/example/sysinfo/odp_sysinfo.c -o odp_sysinfo_inst_dynamic `PKG_CONFIG_PATH=/opt/odp/lib/pkgconfig:${PKG_CONFIG_PATH} pkg-config --cflags --libs libodp-linux` -popd +make installcheck diff --git a/scripts/ci/build_static_x86_64.sh b/scripts/ci/build_static_x86_64.sh index f32873afc..50a056396 100755 --- a/scripts/ci/build_static_x86_64.sh +++ b/scripts/ci/build_static_x86_64.sh @@ -14,22 +14,10 @@ make -j $(nproc) make install -# Build and run sysinfo with installed libs -pushd ${HOME} - -# Default ODP library name -if [ -z "$ODP_LIB_NAME" ] ; then -ODP_LIB_NAME=libodp-dpdk -fi - -${CC} ${CFLAGS} ${OLDPWD}/example/sysinfo/odp_sysinfo.c -static -o odp_sysinfo_inst_static `PKG_CONFIG_PATH=/opt/odp/lib/pkgconfig:${PKG_CONFIG_PATH} pkg-config --cflags --libs --static ${ODP_LIB_NAME}` - -echo 1500 | tee /proc/sys/vm/nr_hugepages +echo 1000 | tee /proc/sys/vm/nr_hugepages mkdir -p /mnt/huge mount -t hugetlbfs nodev /mnt/huge -./odp_sysinfo_inst_static +make installcheck umount /mnt/huge - -popd diff --git a/test/common/Makefile.am b/test/common/Makefile.am index 745b1d16a..694b442c2 100644 --- a/test/common/Makefile.am +++ b/test/common/Makefile.am @@ -20,5 +20,3 @@ noinst_HEADERS = test_common_macros.h \ test_packet_ipv4.h \ test_packet_ipv4_with_crc.h \ test_packet_ipv6.h - -dist_test_SCRIPTS = run-test.sh diff --git a/test/common/run-test.sh b/test/common/run-test.sh deleted file mode 100755 index 2bff651cc..000000000 --- a/test/common/run-test.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -# -# Run the ODP test applications and report status in a format that -# matches the automake "make check" output. -# -# The list of tests to be run is obtained by sourcing a file that -# contains an environment variable in the form; -# -# TEST="test_app1 test_app2" -# -# The default behaviour is to run all the tests defined in files -# named tests-*.env in the same directory as this script, but a single -# test definition file can be specified using the TEST_DEF environment -# variable. -# -# Test definition files may optionally also specify a LOG_COMPILER -# which will be invoked as a wrapper to each of the test application -# (as per automake). -# -TDIR=$(dirname $(readlink -f $0)) -PASS=0 -FAIL=0 -SKIP=0 -res=0 - -if [ "$V" != "0" ]; then - verbose=1 -else - verbose=0 - mkdir -p logs -fi - -do_run_tests() { - source $1 - - for tc in $TESTS; do - tc=$(basename $tc) - if [ "$verbose" = "0" ]; then - logfile=logs/${tc}.log - touch $logfile || logfile=/dev/null - $LOG_COMPILER $TDIR/$tc > $logfile 2>&1 - else - $LOG_COMPILER $TDIR/$tc - fi - - tres=$? - case $tres in - 0) echo "PASS: $tc"; let PASS=$PASS+1 ;; - 77) echo "SKIP: $tc"; let SKIP=$SKIP+1 ;; - *) echo "FAIL: $tc"; let FAIL=$FAIL+1; res=1 ;; - esac - done -} - -if [ "$TEST_DEFS" != "" -a -f "$TEST_DEFS" ]; then - do_run_tests $TEST_DEFS -elif [ "$1" != "" ]; then - do_run_tests $TDIR/tests-${1}.env -else - for tenv in $TDIR/tests-*.env; do - do_run_tests $tenv - done -fi - -echo "TEST RESULT: $PASS tests passed, $SKIP skipped, $FAIL failed" - -exit $res diff --git a/test/miscellaneous/Makefile.am b/test/miscellaneous/Makefile.am index e8bcf48de..c728784ee 100644 --- a/test/miscellaneous/Makefile.am +++ b/test/miscellaneous/Makefile.am @@ -13,3 +13,36 @@ if ODP_ABI_COMPAT odp_api_headers_CFLAGS += -Wconversion endif odp_api_headers_SOURCES = odp_api_headers.c + +PKGCONFIG = PKG_CONFIG_PATH=$(libdir)/pkgconfig:$$PKG_CONFIG_PATH pkg-config --cflags --libs + +if enable_shared + +PROGRAM_shared = odp_api_headers_shared + +installcheck-local: $(PROGRAM_shared) + +$(PROGRAM_shared): $(srcdir)/$(odp_api_headers_SOURCES) + $(CC) $(AM_CFLAGS) $(CFLAGS) $^ -o $@ \ + `$(PKGCONFIG) libodphelper` `$(PKGCONFIG) lib$(ODP_LIB_NAME)` +if ! cross_compile + LD_LIBRARY_PATH=$(libdir) ./$@ +endif +endif + +if enable_static + +PROGRAM_static = odp_api_headers_static + +installcheck-local: $(PROGRAM_static) + +$(PROGRAM_static): $(srcdir)/$(odp_api_headers_SOURCES) + $(CC) $(AM_CFLAGS) $(CFLAGS) $^ -o $@ \ + `$(PKGCONFIG) --static libodphelper | sed "s/-lodphelper/-l:libodphelper.a/"` \ + `$(PKGCONFIG) --static lib$(ODP_LIB_NAME) | sed "s/-l$(ODP_LIB_NAME)/-l:lib$(ODP_LIB_NAME).a/"` +if ! cross_compile + ./$@ +endif +endif + +DISTCLEANFILES = $(PROGRAM_shared) $(PROGRAM_static) diff --git a/test/miscellaneous/odp_api_headers.c b/test/miscellaneous/odp_api_headers.c index f81a9c58b..843de2a02 100644 --- a/test/miscellaneous/odp_api_headers.c +++ b/test/miscellaneous/odp_api_headers.c @@ -10,5 +10,30 @@ int main(void) { + odp_instance_t inst; + + if (odp_init_global(&inst, NULL, NULL)) { + ODPH_ERR("Global init failed.\n"); + return -1; + } + + if (odp_init_local(inst, ODP_THREAD_CONTROL)) { + ODPH_ERR("Local init failed.\n"); + return -1; + } + + odp_sys_info_print(); + printf("Helper library version: %s\n", odph_version_str()); + + if (odp_term_local()) { + ODPH_ERR("Local term failed.\n"); + return -1; + } + + if (odp_term_global(inst)) { + ODPH_ERR("Global term failed.\n"); + return -1; + } + return 0; } diff --git a/test/performance/.gitignore b/test/performance/.gitignore index 22abea846..3a32d0420 100644 --- a/test/performance/.gitignore +++ b/test/performance/.gitignore @@ -2,6 +2,7 @@ *.trs odp_atomic odp_atomic_perf +odp_bench_buffer odp_bench_packet odp_cpu_bench odp_crc diff --git a/test/performance/Makefile.am b/test/performance/Makefile.am index c22980e11..b0885808e 100644 --- a/test/performance/Makefile.am +++ b/test/performance/Makefile.am @@ -3,6 +3,7 @@ include $(top_srcdir)/test/Makefile.inc TESTS_ENVIRONMENT += TEST_DIR=${builddir} EXECUTABLES = odp_atomic_perf \ + odp_bench_buffer \ odp_bench_packet \ odp_crc \ odp_lock_perf \ @@ -51,6 +52,7 @@ endif bin_PROGRAMS = $(EXECUTABLES) $(COMPILE_ONLY) odp_atomic_perf_SOURCES = odp_atomic_perf.c +odp_bench_buffer_SOURCES = odp_bench_buffer.c odp_bench_packet_SOURCES = odp_bench_packet.c odp_cpu_bench_SOURCES = odp_cpu_bench.c odp_crc_SOURCES = odp_crc.c @@ -73,9 +75,8 @@ odp_sched_perf_SOURCES = odp_sched_perf.c odp_timer_perf_SOURCES = odp_timer_perf.c # l2fwd test depends on generator example -EXTRA_odp_l2fwd_DEPENDENCIES = example-generator -.PHONY: $(EXTRA_odp_l2fwd_DEPENDENCIES) -example-generator: +EXTRA_odp_l2fwd_DEPENDENCIES = $(top_builddir)/example/generator/odp_generator$(EXEEXT) +$(top_builddir)/example/generator/odp_generator$(EXEEXT): $(MAKE) -C $(top_builddir)/example/generator odp_generator$(EXEEXT) dist_check_SCRIPTS = $(TESTSCRIPTS) diff --git a/test/performance/odp_bench_buffer.c b/test/performance/odp_bench_buffer.c new file mode 100644 index 000000000..e7029f57d --- /dev/null +++ b/test/performance/odp_bench_buffer.c @@ -0,0 +1,920 @@ +/* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_api.h> +#include <odp/helper/odph_api.h> + +#include <getopt.h> +#include <inttypes.h> +#include <signal.h> +#include <stdlib.h> +#include <unistd.h> + +/** Default buffer size */ +#define TEST_BUF_SIZE 1024 + +/** Default pool user area size in bytes */ +#define TEST_UAREA_SIZE 8 + +/** Number of function calls per test cycle */ +#define TEST_REPEAT_COUNT 1000 + +/** Default number of test cycles */ +#define TEST_CYCLES 1000 + +/** Maximum burst size for *_multi operations */ +#define TEST_MAX_BURST 64 + +/** Default burst size for *_multi operations */ +#define TEST_DEF_BURST 8 + +/** Get rid of path in filename - only for unix-type paths using '/' */ +#define NO_PATH(file_name) (strrchr((file_name), '/') ? \ + strrchr((file_name), '/') + 1 : (file_name)) + +#define BENCH_INFO(run, init, term, name) \ + {#run, run, init, term, name, NULL} + +#define BENCH_INFO_COND(run, init, term, name, cond) \ + {#run, run, init, term, name, cond} + +/** + * Parsed command line arguments + */ +typedef struct { + int bench_idx; /** Benchmark index to run indefinitely */ + int burst_size; /** Burst size for *_multi operations */ + int cache_size; /** Pool cache size */ + int test_cycles; /** Test cycles per tested function */ +} appl_args_t; + +/** + * Initialize benchmark resources + */ +typedef void (*bench_init_fn_t)(void); + +/** + * Run benchmark + * + * @retval >0 on success + * */ +typedef int (*bench_run_fn_t)(void); + +/** + * Release benchmark resources + */ +typedef void (*bench_term_fn_t)(void); + +/** + * Check benchmark preconditions + * + * @retval !0 test enabled + * */ +typedef int (*bench_cond_fn_t)(void); + +/** + * Benchmark data + */ +typedef struct { + const char *name; + bench_run_fn_t run; + bench_init_fn_t init; + bench_term_fn_t term; + const char *desc; + bench_cond_fn_t cond; +} bench_info_t; + +/** + * Grouping of all global data + */ +typedef struct { + /** Application (parsed) arguments */ + appl_args_t appl; + /** Buffer pool */ + odp_pool_t pool; + /** Benchmark functions */ + bench_info_t *bench; + /** Number of benchmark functions */ + int num_bench; + /** Buffer size */ + uint32_t buf_size; + /** Buffer user area size */ + uint32_t uarea_size; + /** Max flow id */ + uint32_t max_flow_id; + /** Break worker loop if set to 1 */ + odp_atomic_u32_t exit_thread; + /** Array for storing test buffers */ + odp_buffer_t buf_tbl[TEST_REPEAT_COUNT * TEST_MAX_BURST]; + /** Array for storing test event */ + odp_event_t event_tbl[TEST_REPEAT_COUNT * TEST_MAX_BURST]; + /** Array for storing test pointers */ + void *ptr_tbl[TEST_REPEAT_COUNT]; + /** Array for storing test pool handles */ + odp_pool_t pool_tbl[TEST_REPEAT_COUNT]; + /** Array for storing test event types */ + odp_event_type_t event_type_tbl[TEST_REPEAT_COUNT]; + /** Array for storing test event subtypes */ + odp_event_subtype_t event_subtype_tbl[TEST_REPEAT_COUNT]; + /** Benchmark run failed */ + uint8_t bench_failed; + /** CPU mask as string */ + char cpumask_str[ODP_CPUMASK_STR_SIZE]; +} args_t; + +/** Global pointer to args */ +static args_t *gbl_args; + +static void sig_handler(int signo ODP_UNUSED) +{ + if (gbl_args == NULL) + return; + odp_atomic_store_u32(&gbl_args->exit_thread, 1); +} + +/** + * Run given benchmark indefinitely + */ +static void run_indef(args_t *args, int idx) +{ + const char *desc; + + desc = args->bench[idx].desc != NULL ? + args->bench[idx].desc : args->bench[idx].name; + + printf("Running odp_%s test indefinitely\n", desc); + + while (!odp_atomic_load_u32(&gbl_args->exit_thread)) { + int ret; + + if (args->bench[idx].init != NULL) + args->bench[idx].init(); + + ret = args->bench[idx].run(); + + if (args->bench[idx].term != NULL) + args->bench[idx].term(); + + if (!ret) + ODPH_ABORT("Benchmark %s failed\n", desc); + } +} + +static int run_benchmarks(void *arg) +{ + int i, j, k; + args_t *args = arg; + + printf("\nAverage CPU cycles per function call\n" + "---------------------------------------------\n"); + + /* Run each test twice. Results from the first warm-up round are ignored. */ + for (i = 0; i < 2; i++) { + uint64_t tot_cycles = 0; + + for (j = 0, k = 1; j < gbl_args->num_bench; k++) { + int ret; + uint64_t c1, c2; + const char *desc; + + /* Run selected test indefinitely */ + if (args->appl.bench_idx && + (j + 1) != args->appl.bench_idx) { + j++; + continue; + } else if (args->appl.bench_idx && + (j + 1) == args->appl.bench_idx) { + run_indef(args, j); + return 0; + } + + desc = args->bench[j].desc != NULL ? + args->bench[j].desc : + args->bench[j].name; + + /* Skip unsupported tests */ + if (args->bench[j].cond != NULL && !args->bench[j].cond()) { + j++; + k = 0; + if (i > 0) + printf("[%02d] odp_%-26s: n/a\n", j, desc); + continue; + } + + if (args->bench[j].init != NULL) + args->bench[j].init(); + + c1 = odp_cpu_cycles(); + ret = args->bench[j].run(); + c2 = odp_cpu_cycles(); + + if (args->bench[j].term != NULL) + args->bench[j].term(); + + if (!ret) { + ODPH_ERR("Benchmark odp_%s failed\n", desc); + args->bench_failed = 1; + return -1; + } + + tot_cycles += odp_cpu_cycles_diff(c2, c1); + + if (k >= args->appl.test_cycles) { + double cycles; + + /** Each benchmark runs internally TEST_REPEAT_COUNT times. */ + cycles = ((double)tot_cycles) / + (args->appl.test_cycles * + TEST_REPEAT_COUNT); + + /* No print from warm-up round */ + if (i > 0) + printf("[%02d] odp_%-26s: %8.1f\n", j + 1, desc, cycles); + + j++; + k = 0; + tot_cycles = 0; + } + } + } + printf("\n"); + + return 0; +} + +static void allocate_test_buffers(odp_buffer_t buf[], int num) +{ + int num_buf = 0; + + while (num_buf < num) { + int ret; + + ret = odp_buffer_alloc_multi(gbl_args->pool, &buf[num_buf], num - num_buf); + if (ret < 0) + ODPH_ABORT("Allocating test buffers failed\n"); + + num_buf += ret; + } +} + +static void alloc_buffers_multi(void) +{ + allocate_test_buffers(gbl_args->buf_tbl, TEST_REPEAT_COUNT * gbl_args->appl.burst_size); +} + +static void create_buffers(void) +{ + allocate_test_buffers(gbl_args->buf_tbl, TEST_REPEAT_COUNT); +} + +static void create_events(void) +{ + odp_buffer_t *buf_tbl = gbl_args->buf_tbl; + + allocate_test_buffers(gbl_args->buf_tbl, TEST_REPEAT_COUNT); + + for (int i = 0; i < TEST_REPEAT_COUNT; i++) + gbl_args->event_tbl[i] = odp_buffer_to_event(buf_tbl[i]); +} + +static void create_events_multi(void) +{ + odp_buffer_t *buf_tbl = gbl_args->buf_tbl; + + allocate_test_buffers(gbl_args->buf_tbl, + TEST_REPEAT_COUNT * gbl_args->appl.burst_size); + + for (int i = 0; i < TEST_REPEAT_COUNT * gbl_args->appl.burst_size; i++) + gbl_args->event_tbl[i] = odp_buffer_to_event(buf_tbl[i]); +} + +static void free_buffers(void) +{ + odp_buffer_free_multi(gbl_args->buf_tbl, TEST_REPEAT_COUNT); +} + +static void free_buffers_multi(void) +{ + odp_buffer_free_multi(gbl_args->buf_tbl, TEST_REPEAT_COUNT * gbl_args->appl.burst_size); +} + +static int check_uarea(void) +{ + return !!gbl_args->uarea_size; +} + +static int check_flow_aware(void) +{ + return !!gbl_args->max_flow_id; +} + +static int buffer_from_event(void) +{ + odp_buffer_t *buf_tbl = gbl_args->buf_tbl; + odp_event_t *event_tbl = gbl_args->event_tbl; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + buf_tbl[i] = odp_buffer_from_event(event_tbl[i]); + + return i; +} + +static int buffer_to_event(void) +{ + odp_buffer_t *buf_tbl = gbl_args->buf_tbl; + odp_event_t *event_tbl = gbl_args->event_tbl; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + event_tbl[i] = odp_buffer_to_event(buf_tbl[i]); + + return i; +} + +static int buffer_addr(void) +{ + odp_buffer_t *buf_tbl = gbl_args->buf_tbl; + void **ptr_tbl = gbl_args->ptr_tbl; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + ptr_tbl[i] = odp_buffer_addr(buf_tbl[i]); + + return i; +} + +static int buffer_size(void) +{ + odp_buffer_t *buf_tbl = gbl_args->buf_tbl; + uint32_t ret = 0; + + for (int i = 0; i < TEST_REPEAT_COUNT; i++) + ret += odp_buffer_size(buf_tbl[i]); + + return ret; +} + +static int buffer_user_area(void) +{ + odp_buffer_t *buf_tbl = gbl_args->buf_tbl; + void **ptr_tbl = gbl_args->ptr_tbl; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + ptr_tbl[i] = odp_buffer_user_area(buf_tbl[i]); + + return i; +} + +static int buffer_pool(void) +{ + odp_buffer_t *buf_tbl = gbl_args->buf_tbl; + odp_pool_t *pool_tbl = gbl_args->pool_tbl; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + pool_tbl[i] = odp_buffer_pool(buf_tbl[i]); + + return i; +} + +static int buffer_alloc(void) +{ + odp_buffer_t *buf_tbl = gbl_args->buf_tbl; + odp_pool_t pool = gbl_args->pool; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + buf_tbl[i] = odp_buffer_alloc(pool); + + return i; +} + +static int buffer_alloc_multi(void) +{ + odp_buffer_t *buf_tbl = gbl_args->buf_tbl; + odp_pool_t pool = gbl_args->pool; + int burst_size = gbl_args->appl.burst_size; + int num = 0; + + for (int i = 0; i < TEST_REPEAT_COUNT; i++) + num += odp_buffer_alloc_multi(pool, &buf_tbl[num], burst_size); + + return num; +} + +static int buffer_free(void) +{ + odp_buffer_t *buf_tbl = gbl_args->buf_tbl; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + odp_buffer_free(buf_tbl[i]); + + return i; +} + +static int buffer_free_multi(void) +{ + odp_buffer_t *buf_tbl = gbl_args->buf_tbl; + int burst_size = gbl_args->appl.burst_size; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + odp_buffer_free_multi(&buf_tbl[i * burst_size], burst_size); + + return i; +} + +static int buffer_alloc_free(void) +{ + odp_pool_t pool = gbl_args->pool; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) { + odp_buffer_t buf = odp_buffer_alloc(pool); + + if (odp_unlikely(buf == ODP_BUFFER_INVALID)) + return 0; + + odp_buffer_free(buf); + } + return i; +} + +static int buffer_alloc_free_multi(void) +{ + odp_buffer_t *buf_tbl = gbl_args->buf_tbl; + odp_pool_t pool = gbl_args->pool; + int burst_size = gbl_args->appl.burst_size; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) { + int num = odp_buffer_alloc_multi(pool, buf_tbl, burst_size); + + if (odp_unlikely(num < 1)) + return 0; + + odp_buffer_free_multi(buf_tbl, num); + } + return i; +} + +static int buffer_is_valid(void) +{ + odp_buffer_t *buf_tbl = gbl_args->buf_tbl; + uint32_t ret = 0; + + for (int i = 0; i < TEST_REPEAT_COUNT; i++) + ret += odp_buffer_is_valid(buf_tbl[i]); + + return ret; +} + +static int event_type(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + odp_event_type_t *event_type_tbl = gbl_args->event_type_tbl; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + event_type_tbl[i] = odp_event_type(event_tbl[i]); + + return i; +} + +static int event_subtype(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + odp_event_subtype_t *event_subtype_tbl = gbl_args->event_subtype_tbl; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + event_subtype_tbl[i] = odp_event_subtype(event_tbl[i]); + + return i; +} + +static int event_types(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + odp_event_type_t *event_type_tbl = gbl_args->event_type_tbl; + odp_event_subtype_t *event_subtype_tbl = gbl_args->event_subtype_tbl; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + event_type_tbl[i] = odp_event_types(event_tbl[i], &event_subtype_tbl[i]); + + return i; +} + +static int event_type_multi(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + odp_event_type_t *event_type_tbl = gbl_args->event_type_tbl; + int burst_size = gbl_args->appl.burst_size; + uint32_t ret = 0; + + for (int i = 0; i < TEST_REPEAT_COUNT; i++) + ret += odp_event_type_multi(&event_tbl[i * burst_size], burst_size, + &event_type_tbl[i]); + + return ret; +} + +static int event_is_valid(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + + uint32_t ret = 0; + + for (int i = 0; i < TEST_REPEAT_COUNT; i++) + ret += odp_event_is_valid(event_tbl[i]); + + return ret; +} + +static int event_free(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + odp_event_free(event_tbl[i]); + + return i; +} + +static int event_free_multi(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + int burst_size = gbl_args->appl.burst_size; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + odp_event_free_multi(&event_tbl[i * burst_size], burst_size); + + return i; +} + +static int event_free_sp(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + int burst_size = gbl_args->appl.burst_size; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + odp_event_free_sp(&event_tbl[i * burst_size], burst_size); + + return i; +} + +static int event_flow_id(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + uint32_t ret = 0; + + for (int i = 0; i < TEST_REPEAT_COUNT; i++) + ret += odp_event_flow_id(event_tbl[i]); + + return !ret; +} + +static int event_flow_id_set(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + int i = 0; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + odp_event_flow_id_set(event_tbl[i], 0); + + return i; +} + +/** + * Print usage information + */ +static void usage(char *progname) +{ + printf("\n" + "OpenDataPlane Buffer/Event API microbenchmarks.\n" + "\n" + "Usage: %s OPTIONS\n" + " E.g. %s\n" + "\n" + "Optional OPTIONS:\n" + " -b, --burst <num> Test burst size.\n" + " -c, --cache_size <num> Pool cache size.\n" + " -i, --index <idx> Benchmark index to run indefinitely.\n" + " -t, --test_cycles <num> Run each test 'num' times (default %d).\n" + " -h, --help Display help and exit.\n\n" + "\n", NO_PATH(progname), NO_PATH(progname), TEST_CYCLES); +} + +/** + * Parse and store the command line arguments + * + * @param argc argument count + * @param argv[] argument vector + * @param appl_args Store application arguments here + */ +static void parse_args(int argc, char *argv[], appl_args_t *appl_args) +{ + int opt; + int long_index; + static const struct option longopts[] = { + {"burst", required_argument, NULL, 'b'}, + {"cache_size", required_argument, NULL, 'c'}, + {"index", required_argument, NULL, 'i'}, + {"test_cycles", required_argument, NULL, 't'}, + {"help", no_argument, NULL, 'h'}, + {NULL, 0, NULL, 0} + }; + + static const char *shortopts = "c:b:i:t:h"; + + appl_args->bench_idx = 0; /* Run all benchmarks */ + appl_args->burst_size = TEST_DEF_BURST; + appl_args->cache_size = -1; + appl_args->test_cycles = TEST_CYCLES; + + while (1) { + opt = getopt_long(argc, argv, shortopts, longopts, &long_index); + + if (opt == -1) + break; /* No more options */ + + switch (opt) { + case 'c': + appl_args->cache_size = atoi(optarg); + break; + case 'b': + appl_args->burst_size = atoi(optarg); + break; + case 'h': + usage(argv[0]); + exit(EXIT_SUCCESS); + break; + case 'i': + appl_args->bench_idx = atoi(optarg); + break; + case 't': + appl_args->test_cycles = atoi(optarg); + break; + default: + break; + } + } + + if (appl_args->burst_size < 1 || + appl_args->burst_size > TEST_MAX_BURST) { + printf("Invalid burst size (max %d)\n", TEST_MAX_BURST); + exit(EXIT_FAILURE); + } + + if (appl_args->test_cycles < 1) { + printf("Invalid test cycle repeat count: %d\n", appl_args->test_cycles); + exit(EXIT_FAILURE); + } + + optind = 1; /* Reset 'extern optind' from the getopt lib */ +} + +/** + * Print system and application info + */ +static void print_info(void) +{ + odp_sys_info_print(); + + printf("\n" + "odp_bench_buffer options\n" + "------------------------\n"); + + printf("Burst size: %d\n", gbl_args->appl.burst_size); + printf("Buffer size: %d\n", gbl_args->buf_size); + printf("CPU mask: %s\n", gbl_args->cpumask_str); + if (gbl_args->appl.cache_size < 0) + printf("Pool cache size: default\n"); + else + printf("Pool cache size: %d\n", gbl_args->appl.cache_size); + printf("Test cycles: %d\n", gbl_args->appl.test_cycles); + printf("\n"); +} + +/** + * Test functions + */ +bench_info_t test_suite[] = { + BENCH_INFO(buffer_from_event, create_events, free_buffers, NULL), + BENCH_INFO(buffer_to_event, create_buffers, free_buffers, NULL), + BENCH_INFO(buffer_addr, create_buffers, free_buffers, NULL), + BENCH_INFO(buffer_size, create_buffers, free_buffers, NULL), + BENCH_INFO_COND(buffer_user_area, create_buffers, free_buffers, NULL, check_uarea), + BENCH_INFO(buffer_pool, create_buffers, free_buffers, NULL), + BENCH_INFO(buffer_alloc, NULL, free_buffers, NULL), + BENCH_INFO(buffer_alloc_multi, NULL, free_buffers_multi, NULL), + BENCH_INFO(buffer_free, create_buffers, NULL, NULL), + BENCH_INFO(buffer_free_multi, alloc_buffers_multi, NULL, NULL), + BENCH_INFO(buffer_alloc_free, NULL, NULL, NULL), + BENCH_INFO(buffer_alloc_free_multi, NULL, NULL, NULL), + BENCH_INFO(buffer_is_valid, create_buffers, free_buffers, NULL), + BENCH_INFO(event_type, create_events, free_buffers, NULL), + BENCH_INFO(event_subtype, create_buffers, free_buffers, NULL), + BENCH_INFO(event_types, create_buffers, free_buffers, NULL), + BENCH_INFO(event_type_multi, create_events_multi, free_buffers_multi, NULL), + BENCH_INFO(event_is_valid, create_events, free_buffers, NULL), + BENCH_INFO(event_free, create_events, NULL, NULL), + BENCH_INFO(event_free_multi, create_events_multi, NULL, NULL), + BENCH_INFO(event_free_sp, create_events_multi, NULL, NULL), + BENCH_INFO_COND(event_flow_id, create_events, free_buffers, NULL, check_flow_aware), + BENCH_INFO_COND(event_flow_id_set, create_events, free_buffers, NULL, check_flow_aware), +}; + +/** + * ODP buffer microbenchmark application + */ +int main(int argc, char *argv[]) +{ + odph_helper_options_t helper_options; + odph_thread_t worker_thread; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; + int cpu; + odp_shm_t shm; + odp_cpumask_t cpumask, default_mask; + odp_schedule_capability_t sched_capa; + odp_pool_capability_t capa; + odp_pool_param_t params; + odp_instance_t instance; + odp_init_t init_param; + uint32_t buf_num; + uint8_t ret; + + /* Let helper collect its own arguments (e.g. --odph_proc) */ + argc = odph_parse_options(argc, argv); + if (odph_options(&helper_options)) { + ODPH_ERR("Error: reading ODP helper options failed\n"); + exit(EXIT_FAILURE); + } + + odp_init_param_init(&init_param); + init_param.mem_model = helper_options.mem_model; + + /* Init ODP before calling anything else */ + if (odp_init_global(&instance, &init_param, NULL)) { + ODPH_ERR("Error: ODP global init failed\n"); + exit(EXIT_FAILURE); + } + + /* Init this thread */ + if (odp_init_local(instance, ODP_THREAD_CONTROL)) { + ODPH_ERR("Error: ODP local init failed\n"); + exit(EXIT_FAILURE); + } + + /* Reserve memory for args from shared mem */ + shm = odp_shm_reserve("shm_args", sizeof(args_t), ODP_CACHE_LINE_SIZE, 0); + if (shm == ODP_SHM_INVALID) { + ODPH_ERR("Error: shared mem reserve failed\n"); + exit(EXIT_FAILURE); + } + + gbl_args = odp_shm_addr(shm); + if (gbl_args == NULL) { + ODPH_ERR("Error: shared mem alloc failed\n"); + exit(EXIT_FAILURE); + } + + memset(gbl_args, 0, sizeof(args_t)); + odp_atomic_init_u32(&gbl_args->exit_thread, 0); + + gbl_args->bench = test_suite; + gbl_args->num_bench = sizeof(test_suite) / sizeof(test_suite[0]); + + /* Parse and store the application arguments */ + parse_args(argc, argv, &gbl_args->appl); + + /* Get default worker cpumask */ + if (odp_cpumask_default_worker(&default_mask, 1) != 1) { + ODPH_ERR("Error: unable to allocate worker thread\n"); + exit(EXIT_FAILURE); + } + (void)odp_cpumask_to_str(&default_mask, gbl_args->cpumask_str, + sizeof(gbl_args->cpumask_str)); + + if (odp_schedule_capability(&sched_capa)) { + ODPH_ERR("Error: schedule capability failed\n"); + exit(EXIT_FAILURE); + } + + gbl_args->max_flow_id = 0; + if (sched_capa.max_flow_id) { + odp_schedule_config_t sched_config; + + odp_schedule_config_init(&sched_config); + sched_config.max_flow_id = 1; + + if (odp_schedule_config(&sched_config)) { + ODPH_ERR("Error: schedule config failed\n"); + exit(EXIT_FAILURE); + } + gbl_args->max_flow_id = 1; + } + + if (odp_pool_capability(&capa)) { + ODPH_ERR("Error: unable to query pool capability\n"); + exit(EXIT_FAILURE); + } + + buf_num = gbl_args->appl.burst_size * TEST_REPEAT_COUNT; + + if (capa.buf.max_num && capa.buf.max_num < buf_num) { + ODPH_ERR("Error: pool size not supported (max %" PRIu32 ")\n", capa.buf.max_num); + exit(EXIT_FAILURE); + } else if (gbl_args->appl.cache_size > (int)capa.buf.max_cache_size) { + ODPH_ERR("Error: cache size not supported (max %" PRIu32 ")\n", + capa.buf.max_cache_size); + exit(EXIT_FAILURE); + } + + gbl_args->buf_size = TEST_BUF_SIZE; + if (capa.buf.max_size && capa.buf.max_size < TEST_BUF_SIZE) + gbl_args->buf_size = capa.buf.max_size; + + gbl_args->uarea_size = TEST_UAREA_SIZE < capa.buf.max_uarea_size ? + TEST_UAREA_SIZE : capa.buf.max_uarea_size; + + print_info(); + + /* Create buffer pool */ + odp_pool_param_init(¶ms); + params.buf.size = gbl_args->buf_size; + params.buf.num = buf_num; + params.buf.uarea_size = gbl_args->uarea_size; + if (gbl_args->appl.cache_size >= 0) + params.buf.cache_size = gbl_args->appl.cache_size; + params.type = ODP_POOL_BUFFER; + + gbl_args->pool = odp_pool_create("microbench", ¶ms); + if (gbl_args->pool == ODP_POOL_INVALID) { + ODPH_ERR("Error: pool create failed\n"); + exit(EXIT_FAILURE); + } + + odp_pool_print(gbl_args->pool); + + memset(&worker_thread, 0, sizeof(odph_thread_t)); + + signal(SIGINT, sig_handler); + + /* Create worker thread */ + cpu = odp_cpumask_first(&default_mask); + + odp_cpumask_zero(&cpumask); + odp_cpumask_set(&cpumask, cpu); + + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &cpumask; + thr_common.share_param = 1; + + odph_thread_param_init(&thr_param); + thr_param.start = run_benchmarks; + thr_param.arg = gbl_args; + thr_param.thr_type = ODP_THREAD_WORKER; + + odph_thread_create(&worker_thread, &thr_common, &thr_param, 1); + + odph_thread_join(&worker_thread, 1); + + ret = gbl_args->bench_failed; + + if (odp_pool_destroy(gbl_args->pool)) { + ODPH_ERR("Error: pool destroy\n"); + exit(EXIT_FAILURE); + } + + if (odp_shm_free(shm)) { + ODPH_ERR("Error: shm free\n"); + exit(EXIT_FAILURE); + } + + if (odp_term_local()) { + ODPH_ERR("Error: term local\n"); + exit(EXIT_FAILURE); + } + + if (odp_term_global(instance)) { + ODPH_ERR("Error: term global\n"); + exit(EXIT_FAILURE); + } + + return ret; +} diff --git a/test/performance/odp_bench_packet.c b/test/performance/odp_bench_packet.c index 23fd17bea..b44e92b72 100644 --- a/test/performance/odp_bench_packet.c +++ b/test/performance/odp_bench_packet.c @@ -1,4 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -84,6 +85,7 @@ const uint32_t test_packet_len[] = {WARM_UP, TEST_MIN_PKT_SIZE, 128, 256, 512, typedef struct { int bench_idx; /** Benchmark index to run indefinitely */ int burst_size; /** Burst size for *_multi operations */ + int cache_size; /** Pool cache size */ } appl_args_t; /** @@ -143,7 +145,7 @@ typedef struct { /** Array for storing test packets */ odp_packet_t pkt2_tbl[TEST_REPEAT_COUNT]; /** Array for storing test event */ - odp_event_t event_tbl[TEST_REPEAT_COUNT]; + odp_event_t event_tbl[TEST_REPEAT_COUNT * TEST_MAX_BURST]; /** Array for storing test pointers */ void *ptr_tbl[TEST_REPEAT_COUNT]; /** Array for storing test segments */ @@ -182,7 +184,7 @@ static void run_indef(args_t *args, int idx) desc = args->bench[idx].desc != NULL ? args->bench[idx].desc : args->bench[idx].name; - printf("Running %s() indefinitely\n", desc); + printf("Running odp_%s test indefinitely\n", desc); while (!odp_atomic_load_u32(&gbl_args->exit_thread)) { int ret; @@ -270,7 +272,7 @@ static int run_benchmarks(void *arg) TEST_REPEAT_COUNT); results[j][i] = cycles; - printf("%-30s: %8.1f\n", desc, cycles); + printf("odp_%-26s: %8.1f\n", desc, cycles); j++; k = 0; @@ -290,7 +292,7 @@ static int run_benchmarks(void *arg) printf("----------"); for (i = 0; i < gbl_args->num_bench; i++) { - printf("\n[%02d] %-30s", i + 1, args->bench[i].desc != NULL ? + printf("\n[%02d] odp_%-26s", i + 1, args->bench[i].desc != NULL ? args->bench[i].desc : args->bench[i].name); for (j = 0; j < num_sizes; j++) @@ -516,17 +518,7 @@ static void free_packets_twice(void) odp_packet_free_multi(gbl_args->pkt2_tbl, TEST_REPEAT_COUNT); } -static int bench_empty(void) -{ - int i; - - for (i = 0; i < TEST_REPEAT_COUNT; i++) - gbl_args->output_tbl[i] = i; - - return i; -} - -static int bench_packet_alloc(void) +static int packet_alloc(void) { int i; @@ -541,7 +533,7 @@ static int bench_packet_alloc(void) return i; } -static int bench_packet_alloc_multi(void) +static int packet_alloc_multi(void) { int i; int pkts = 0; @@ -554,7 +546,7 @@ static int bench_packet_alloc_multi(void) return pkts; } -static int bench_packet_free(void) +static int packet_free(void) { int i; @@ -564,7 +556,7 @@ static int bench_packet_free(void) return i; } -static int bench_packet_free_multi(void) +static int packet_free_multi(void) { int i; @@ -577,7 +569,7 @@ static int bench_packet_free_multi(void) return i; } -static int bench_packet_free_sp(void) +static int packet_free_sp(void) { int i; @@ -590,7 +582,7 @@ static int bench_packet_free_sp(void) return i; } -static int bench_packet_alloc_free(void) +static int packet_alloc_free(void) { int i; @@ -604,7 +596,7 @@ static int bench_packet_alloc_free(void) return i; } -static int bench_packet_alloc_free_multi(void) +static int packet_alloc_free_multi(void) { int i; int pkts; @@ -622,7 +614,7 @@ static int bench_packet_alloc_free_multi(void) return i; } -static int bench_packet_reset(void) +static int packet_reset(void) { int i; int ret = 0; @@ -633,7 +625,7 @@ static int bench_packet_reset(void) return !ret; } -static int bench_packet_from_event(void) +static int packet_from_event(void) { int i; odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; @@ -644,7 +636,7 @@ static int bench_packet_from_event(void) return i; } -static int bench_packet_from_event_multi(void) +static int packet_from_event_multi(void) { int i; @@ -658,7 +650,7 @@ static int bench_packet_from_event_multi(void) return i; } -static int bench_packet_to_event(void) +static int packet_to_event(void) { int i; odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; @@ -669,7 +661,7 @@ static int bench_packet_to_event(void) return i; } -static int bench_packet_to_event_multi(void) +static int packet_to_event_multi(void) { int i; @@ -683,7 +675,7 @@ static int bench_packet_to_event_multi(void) return i; } -static int bench_packet_head(void) +static int packet_head(void) { int i; @@ -693,7 +685,7 @@ static int bench_packet_head(void) return i; } -static int bench_packet_buf_len(void) +static int packet_buf_len(void) { int i; uint32_t ret = 0; @@ -704,7 +696,7 @@ static int bench_packet_buf_len(void) return ret; } -static int bench_packet_data(void) +static int packet_data(void) { int i; @@ -714,7 +706,7 @@ static int bench_packet_data(void) return i; } -static int bench_packet_data_seg_len(void) +static int packet_data_seg_len(void) { odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; uint32_t *output_tbl = gbl_args->output_tbl; @@ -726,7 +718,7 @@ static int bench_packet_data_seg_len(void) return i; } -static int bench_packet_seg_len(void) +static int packet_seg_len(void) { int i; uint32_t ret = 0; @@ -737,7 +729,7 @@ static int bench_packet_seg_len(void) return ret; } -static int bench_packet_len(void) +static int packet_len(void) { int i; uint32_t ret = 0; @@ -748,7 +740,7 @@ static int bench_packet_len(void) return ret; } -static int bench_packet_headroom(void) +static int packet_headroom(void) { int i; uint32_t ret = 0; @@ -759,7 +751,7 @@ static int bench_packet_headroom(void) return i + ret; } -static int bench_packet_tailroom(void) +static int packet_tailroom(void) { int i; uint32_t ret = 0; @@ -770,7 +762,7 @@ static int bench_packet_tailroom(void) return i + ret; } -static int bench_packet_tail(void) +static int packet_tail(void) { int i; @@ -780,7 +772,7 @@ static int bench_packet_tail(void) return i; } -static int bench_packet_offset(void) +static int packet_offset(void) { int i; uint32_t offset = gbl_args->pkt.len / 2; @@ -791,7 +783,7 @@ static int bench_packet_offset(void) return i; } -static int bench_packet_prefetch(void) +static int packet_prefetch(void) { int i; @@ -801,7 +793,7 @@ static int bench_packet_prefetch(void) return i; } -static int bench_packet_push_head(void) +static int packet_push_head(void) { int i; odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; @@ -813,7 +805,7 @@ static int bench_packet_push_head(void) return i; } -static int bench_packet_pull_head(void) +static int packet_pull_head(void) { int i; uint32_t len = gbl_args->pkt.seg_len - 1; @@ -825,7 +817,7 @@ static int bench_packet_pull_head(void) return i; } -static int bench_packet_push_tail(void) +static int packet_push_tail(void) { int i; odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; @@ -837,7 +829,7 @@ static int bench_packet_push_tail(void) return i; } -static int bench_packet_pull_tail(void) +static int packet_pull_tail(void) { int i; uint32_t len = gbl_args->pkt.seg_len - 1; @@ -849,7 +841,7 @@ static int bench_packet_pull_tail(void) return i; } -static int bench_packet_extend_head(void) +static int packet_extend_head(void) { int i; int ret = 0; @@ -864,7 +856,7 @@ static int bench_packet_extend_head(void) return ret >= 0; } -static int bench_packet_trunc_head(void) +static int packet_trunc_head(void) { int i; int ret = 0; @@ -879,7 +871,7 @@ static int bench_packet_trunc_head(void) return ret >= 0; } -static int bench_packet_extend_tail(void) +static int packet_extend_tail(void) { int i; int ret = 0; @@ -894,7 +886,7 @@ static int bench_packet_extend_tail(void) return ret >= 0; } -static int bench_packet_trunc_tail(void) +static int packet_trunc_tail(void) { int i; int ret = 0; @@ -909,7 +901,7 @@ static int bench_packet_trunc_tail(void) return ret >= 0; } -static int bench_packet_add_data(void) +static int packet_add_data(void) { int i; int ret = 0; @@ -922,7 +914,7 @@ static int bench_packet_add_data(void) return ret >= 0; } -static int bench_packet_rem_data(void) +static int packet_rem_data(void) { int i; int ret = 0; @@ -935,7 +927,7 @@ static int bench_packet_rem_data(void) return ret >= 0; } -static int bench_packet_align(void) +static int packet_align(void) { int i; int ret = 0; @@ -947,7 +939,7 @@ static int bench_packet_align(void) return ret >= 0; } -static int bench_packet_is_segmented(void) +static int packet_is_segmented(void) { int i; uint32_t ret = 0; @@ -958,7 +950,7 @@ static int bench_packet_is_segmented(void) return (ret == 0) ? 1 : ret; } -static int bench_packet_num_segs(void) +static int packet_num_segs(void) { int i; uint32_t ret = 0; @@ -969,7 +961,7 @@ static int bench_packet_num_segs(void) return ret; } -static int bench_packet_first_seg(void) +static int packet_first_seg(void) { int i; odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; @@ -980,7 +972,7 @@ static int bench_packet_first_seg(void) return i; } -static int bench_packet_last_seg(void) +static int packet_last_seg(void) { int i; odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; @@ -991,7 +983,7 @@ static int bench_packet_last_seg(void) return i; } -static int bench_packet_next_seg(void) +static int packet_next_seg(void) { int i; odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; @@ -1003,7 +995,7 @@ static int bench_packet_next_seg(void) return i; } -static int bench_packet_seg_data(void) +static int packet_seg_data(void) { int i; odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; @@ -1015,7 +1007,7 @@ static int bench_packet_seg_data(void) return i; } -static int bench_packet_seg_data_len(void) +static int packet_seg_data_len(void) { int i; uint32_t ret = 0; @@ -1028,7 +1020,7 @@ static int bench_packet_seg_data_len(void) return ret; } -static int bench_packet_concat(void) +static int packet_concat(void) { int i; int ret = 0; @@ -1041,7 +1033,7 @@ static int bench_packet_concat(void) return ret >= 0; } -static int bench_packet_split(void) +static int packet_split(void) { int i; int ret = 0; @@ -1057,7 +1049,7 @@ static int bench_packet_split(void) return ret >= 0; } -static int bench_packet_copy(void) +static int packet_copy(void) { int i; odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; @@ -1069,7 +1061,7 @@ static int bench_packet_copy(void) return i; } -static int bench_packet_copy_part(void) +static int packet_copy_part(void) { int i; uint32_t len = gbl_args->pkt.len / 2; @@ -1082,7 +1074,7 @@ static int bench_packet_copy_part(void) return i; } -static int bench_packet_copy_to_mem(void) +static int packet_copy_to_mem(void) { int i; uint32_t ret = 0; @@ -1095,7 +1087,7 @@ static int bench_packet_copy_to_mem(void) return !ret; } -static int bench_packet_copy_from_mem(void) +static int packet_copy_from_mem(void) { int i; uint32_t ret = 0; @@ -1108,7 +1100,7 @@ static int bench_packet_copy_from_mem(void) return !ret; } -static int bench_packet_copy_from_pkt(void) +static int packet_copy_from_pkt(void) { int i; uint32_t ret = 0; @@ -1122,7 +1114,7 @@ static int bench_packet_copy_from_pkt(void) return !ret; } -static int bench_packet_copy_data(void) +static int packet_copy_data(void) { int i; uint32_t ret = 0; @@ -1135,7 +1127,7 @@ static int bench_packet_copy_data(void) return !ret; } -static int bench_packet_move_data(void) +static int packet_move_data(void) { int i; uint32_t ret = 0; @@ -1149,7 +1141,7 @@ static int bench_packet_move_data(void) return !ret; } -static int bench_packet_pool(void) +static int packet_pool(void) { int i; @@ -1159,7 +1151,7 @@ static int bench_packet_pool(void) return i; } -static int bench_packet_input(void) +static int packet_input(void) { int i; @@ -1169,7 +1161,7 @@ static int bench_packet_input(void) return i; } -static int bench_packet_input_index(void) +static int packet_input_index(void) { int i; int ret = 0; @@ -1180,7 +1172,7 @@ static int bench_packet_input_index(void) return (ret == 0) ? 1 : ret; } -static int bench_packet_user_ptr(void) +static int packet_user_ptr(void) { int i; odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; @@ -1191,7 +1183,7 @@ static int bench_packet_user_ptr(void) return i; } -static int bench_packet_user_ptr_set(void) +static int packet_user_ptr_set(void) { int i; @@ -1202,7 +1194,7 @@ static int bench_packet_user_ptr_set(void) return i; } -static int bench_packet_user_area(void) +static int packet_user_area(void) { int i; odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; @@ -1213,7 +1205,7 @@ static int bench_packet_user_area(void) return i; } -static int bench_packet_user_area_size(void) +static int packet_user_area_size(void) { int i; uint32_t ret = 0; @@ -1224,7 +1216,7 @@ static int bench_packet_user_area_size(void) return ret; } -static int bench_packet_l2_ptr(void) +static int packet_l2_ptr(void) { int i; @@ -1234,7 +1226,7 @@ static int bench_packet_l2_ptr(void) return i; } -static int bench_packet_l2_offset(void) +static int packet_l2_offset(void) { int i; int ret = 0; @@ -1245,7 +1237,7 @@ static int bench_packet_l2_offset(void) return ret >= 0; } -static int bench_packet_l2_offset_set(void) +static int packet_l2_offset_set(void) { int i; uint32_t ret = 0; @@ -1257,7 +1249,7 @@ static int bench_packet_l2_offset_set(void) return !ret; } -static int bench_packet_l3_ptr(void) +static int packet_l3_ptr(void) { int i; @@ -1267,7 +1259,7 @@ static int bench_packet_l3_ptr(void) return i; } -static int bench_packet_l3_offset(void) +static int packet_l3_offset(void) { int i; int ret = 0; @@ -1278,7 +1270,7 @@ static int bench_packet_l3_offset(void) return ret >= 0; } -static int bench_packet_l3_offset_set(void) +static int packet_l3_offset_set(void) { int i; uint32_t ret = 0; @@ -1290,7 +1282,7 @@ static int bench_packet_l3_offset_set(void) return !ret; } -static int bench_packet_l4_ptr(void) +static int packet_l4_ptr(void) { int i; @@ -1300,7 +1292,7 @@ static int bench_packet_l4_ptr(void) return i; } -static int bench_packet_l4_offset(void) +static int packet_l4_offset(void) { int i; int ret = 0; @@ -1311,7 +1303,7 @@ static int bench_packet_l4_offset(void) return ret >= 0; } -static int bench_packet_l4_offset_set(void) +static int packet_l4_offset_set(void) { int i; uint32_t ret = 0; @@ -1323,7 +1315,7 @@ static int bench_packet_l4_offset_set(void) return !ret; } -static int bench_packet_flow_hash(void) +static int packet_flow_hash(void) { int i; uint32_t ret = 0; @@ -1334,7 +1326,7 @@ static int bench_packet_flow_hash(void) return ret; } -static int bench_packet_flow_hash_set(void) +static int packet_flow_hash_set(void) { int i; @@ -1344,7 +1336,7 @@ static int bench_packet_flow_hash_set(void) return i; } -static int bench_packet_ts(void) +static int packet_ts(void) { int i; @@ -1354,7 +1346,7 @@ static int bench_packet_ts(void) return i; } -static int bench_packet_ts_set(void) +static int packet_ts_set(void) { int i; odp_time_t ts = odp_time_local(); @@ -1365,7 +1357,7 @@ static int bench_packet_ts_set(void) return i; } -static int bench_packet_ref_static(void) +static int packet_ref_static(void) { int i; odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; @@ -1377,7 +1369,7 @@ static int bench_packet_ref_static(void) return i; } -static int bench_packet_ref(void) +static int packet_ref(void) { int i; uint32_t offset = TEST_MIN_PKT_SIZE / 2; @@ -1390,7 +1382,7 @@ static int bench_packet_ref(void) return i; } -static int bench_packet_ref_pkt(void) +static int packet_ref_pkt(void) { int i; uint32_t offset = TEST_MIN_PKT_SIZE / 2; @@ -1403,7 +1395,7 @@ static int bench_packet_ref_pkt(void) return i; } -static int bench_packet_has_ref(void) +static int packet_has_ref(void) { int i; uint32_t ret = 0; @@ -1415,7 +1407,7 @@ static int bench_packet_has_ref(void) return i + ret; } -static int bench_packet_subtype(void) +static int packet_subtype(void) { int i; odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; @@ -1426,7 +1418,7 @@ static int bench_packet_subtype(void) return i; } -static int bench_packet_parse(void) +static int packet_parse(void) { odp_packet_parse_param_t param; odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; @@ -1446,7 +1438,7 @@ static int bench_packet_parse(void) return !ret; } -static int bench_packet_parse_multi(void) +static int packet_parse_multi(void) { int burst_size = gbl_args->appl.burst_size; int ret = 0; @@ -1485,9 +1477,10 @@ static void usage(char *progname) " E.g. %s\n" "\n" "Optional OPTIONS:\n" - " -b, --burst Test packet burst size.\n" - " -i, --index Benchmark index to run indefinitely.\n" - " -h, --help Display help and exit.\n\n" + " -b, --burst <num> Test packet burst size.\n" + " -c, --cache_size <num> Pool cache size.\n" + " -i, --index <idx> Benchmark index to run indefinitely.\n" + " -h, --help Display help and exit.\n\n" "\n", NO_PATH(progname), NO_PATH(progname)); } @@ -1504,15 +1497,17 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) int long_index; static const struct option longopts[] = { {"burst", required_argument, NULL, 'b'}, + {"cache_size", required_argument, NULL, 'c'}, {"help", no_argument, NULL, 'h'}, {"index", required_argument, NULL, 'i'}, {NULL, 0, NULL, 0} }; - static const char *shortopts = "b:i:h"; + static const char *shortopts = "c:b:i:h"; appl_args->bench_idx = 0; /* Run all benchmarks */ appl_args->burst_size = TEST_DEF_BURST; + appl_args->cache_size = -1; while (1) { opt = getopt_long(argc, argv, shortopts, longopts, &long_index); @@ -1521,6 +1516,9 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) break; /* No more options */ switch (opt) { + case 'c': + appl_args->cache_size = atoi(optarg); + break; case 'b': appl_args->burst_size = atoi(optarg); break; @@ -1561,176 +1559,97 @@ static void print_info(char *progname, appl_args_t *appl_args ODP_UNUSED) * Test functions */ bench_info_t test_suite[] = { - BENCH_INFO(bench_empty, NULL, NULL, NULL), - BENCH_INFO(bench_packet_alloc, NULL, free_packets, NULL), - BENCH_INFO(bench_packet_alloc_multi, NULL, free_packets_multi, - NULL), - BENCH_INFO(bench_packet_free, create_packets, NULL, NULL), - BENCH_INFO(bench_packet_free_multi, alloc_packets_multi, NULL, - NULL), - BENCH_INFO(bench_packet_free_sp, alloc_packets_multi, NULL, - NULL), - BENCH_INFO(bench_packet_alloc_free, NULL, NULL, NULL), - BENCH_INFO(bench_packet_alloc_free_multi, NULL, NULL, NULL), - BENCH_INFO(bench_packet_reset, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_from_event, create_events, free_packets, - NULL), - BENCH_INFO(bench_packet_from_event_multi, create_events_multi, - free_packets_multi, NULL), - BENCH_INFO(bench_packet_to_event, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_to_event_multi, alloc_packets_multi, - free_packets_multi, NULL), - BENCH_INFO(bench_packet_head, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_buf_len, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_data, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_data_seg_len, create_packets, - free_packets, NULL), - BENCH_INFO(bench_packet_seg_len, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_len, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_headroom, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_tailroom, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_tail, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_offset, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_prefetch, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_push_head, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_pull_head, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_push_tail, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_pull_tail, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_extend_head, alloc_packets_half, - free_packets, NULL), - BENCH_INFO(bench_packet_trunc_head, create_packets, - free_packets, NULL), - BENCH_INFO(bench_packet_extend_tail, alloc_packets_half, - free_packets, NULL), - BENCH_INFO(bench_packet_trunc_tail, create_packets, - free_packets, NULL), - BENCH_INFO(bench_packet_add_data, alloc_packets_half, - free_packets, NULL), - BENCH_INFO(bench_packet_rem_data, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_align, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_is_segmented, create_packets, - free_packets, NULL), - BENCH_INFO(bench_packet_num_segs, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_first_seg, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_last_seg, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_next_seg, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_seg_data, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_seg_data_len, create_packets, - free_packets, NULL), - BENCH_INFO(bench_packet_concat, alloc_concat_packets, - free_packets, NULL), - BENCH_INFO(bench_packet_split, create_packets, - free_packets_twice, NULL), - BENCH_INFO(bench_packet_copy, create_packets, - free_packets_twice, NULL), - BENCH_INFO(bench_packet_copy_part, create_packets, - free_packets_twice, NULL), - BENCH_INFO(bench_packet_copy_to_mem, create_packets, - free_packets, NULL), - BENCH_INFO(bench_packet_copy_from_mem, create_packets, - free_packets, NULL), - BENCH_INFO(bench_packet_copy_from_pkt, alloc_packets_twice, - free_packets_twice, NULL), - BENCH_INFO(bench_packet_copy_data, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_move_data, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_pool, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_input, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_input_index, create_packets, - free_packets, NULL), - BENCH_INFO(bench_packet_user_ptr, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_user_ptr_set, create_packets, - free_packets, NULL), - BENCH_INFO(bench_packet_user_area, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_user_area_size, create_packets, - free_packets, NULL), - BENCH_INFO(bench_packet_l2_ptr, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_l2_offset, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_l2_offset_set, create_packets, - free_packets, NULL), - BENCH_INFO(bench_packet_l3_ptr, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_l3_offset, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_l3_offset_set, create_packets, - free_packets, NULL), - BENCH_INFO(bench_packet_l4_ptr, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_l4_offset, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_l4_offset_set, create_packets, - free_packets, NULL), - BENCH_INFO(bench_packet_flow_hash, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_flow_hash_set, create_packets, - free_packets, NULL), - BENCH_INFO(bench_packet_ts, create_packets, free_packets, NULL), - BENCH_INFO(bench_packet_ts_set, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_ref_static, create_packets, - free_packets_twice, NULL), - BENCH_INFO(bench_packet_ref, create_packets, - free_packets_twice, NULL), - BENCH_INFO(bench_packet_ref_pkt, alloc_packets_twice, - free_packets_twice, NULL), - BENCH_INFO(bench_packet_has_ref, alloc_ref_packets, - free_packets_twice, NULL), - BENCH_INFO(bench_packet_subtype, create_packets, free_packets, - NULL), - BENCH_INFO(bench_packet_parse, alloc_parse_packets_ipv4_tcp, - free_packets, "bench_packet_parse_ipv4_tcp"), - BENCH_INFO(bench_packet_parse, alloc_parse_packets_ipv4_udp, - free_packets, "bench_packet_parse_ipv4_udp"), - BENCH_INFO(bench_packet_parse, alloc_parse_packets_ipv6_tcp, - free_packets, "bench_packet_parse_ipv6_tcp"), - BENCH_INFO(bench_packet_parse, alloc_parse_packets_ipv6_udp, - free_packets, "bench_packet_parse_ipv6_udp"), - BENCH_INFO(bench_packet_parse_multi, - alloc_parse_packets_multi_ipv4_tcp, - free_packets_multi, - "bench_packet_parse_multi_ipv4_tcp"), - BENCH_INFO(bench_packet_parse_multi, - alloc_parse_packets_multi_ipv4_udp, - free_packets_multi, - "bench_packet_parse_multi_ipv4_udp"), - BENCH_INFO(bench_packet_parse_multi, - alloc_parse_packets_multi_ipv6_tcp, - free_packets_multi, - "bench_packet_parse_multi_ipv6_tcp"), - BENCH_INFO(bench_packet_parse_multi, - alloc_parse_packets_multi_ipv6_udp, - free_packets_multi, - "bench_packet_parse_multi_ipv6_udp"), + BENCH_INFO(packet_alloc, NULL, free_packets, NULL), + BENCH_INFO(packet_alloc_multi, NULL, free_packets_multi, NULL), + BENCH_INFO(packet_free, create_packets, NULL, NULL), + BENCH_INFO(packet_free_multi, alloc_packets_multi, NULL, NULL), + BENCH_INFO(packet_free_sp, alloc_packets_multi, NULL, NULL), + BENCH_INFO(packet_alloc_free, NULL, NULL, NULL), + BENCH_INFO(packet_alloc_free_multi, NULL, NULL, NULL), + BENCH_INFO(packet_reset, create_packets, free_packets, NULL), + BENCH_INFO(packet_from_event, create_events, free_packets, NULL), + BENCH_INFO(packet_from_event_multi, create_events_multi, free_packets_multi, NULL), + BENCH_INFO(packet_to_event, create_packets, free_packets, NULL), + BENCH_INFO(packet_to_event_multi, alloc_packets_multi, free_packets_multi, NULL), + BENCH_INFO(packet_head, create_packets, free_packets, NULL), + BENCH_INFO(packet_buf_len, create_packets, free_packets, NULL), + BENCH_INFO(packet_data, create_packets, free_packets, NULL), + BENCH_INFO(packet_data_seg_len, create_packets, free_packets, NULL), + BENCH_INFO(packet_seg_len, create_packets, free_packets, NULL), + BENCH_INFO(packet_len, create_packets, free_packets, NULL), + BENCH_INFO(packet_headroom, create_packets, free_packets, NULL), + BENCH_INFO(packet_tailroom, create_packets, free_packets, NULL), + BENCH_INFO(packet_tail, create_packets, free_packets, NULL), + BENCH_INFO(packet_offset, create_packets, free_packets, NULL), + BENCH_INFO(packet_prefetch, create_packets, free_packets, NULL), + BENCH_INFO(packet_push_head, create_packets, free_packets, NULL), + BENCH_INFO(packet_pull_head, create_packets, free_packets, NULL), + BENCH_INFO(packet_push_tail, create_packets, free_packets, NULL), + BENCH_INFO(packet_pull_tail, create_packets, free_packets, NULL), + BENCH_INFO(packet_extend_head, alloc_packets_half, free_packets, NULL), + BENCH_INFO(packet_trunc_head, create_packets, free_packets, NULL), + BENCH_INFO(packet_extend_tail, alloc_packets_half, free_packets, NULL), + BENCH_INFO(packet_trunc_tail, create_packets, free_packets, NULL), + BENCH_INFO(packet_add_data, alloc_packets_half, free_packets, NULL), + BENCH_INFO(packet_rem_data, create_packets, free_packets, NULL), + BENCH_INFO(packet_align, create_packets, free_packets, NULL), + BENCH_INFO(packet_is_segmented, create_packets, free_packets, NULL), + BENCH_INFO(packet_num_segs, create_packets, free_packets, NULL), + BENCH_INFO(packet_first_seg, create_packets, free_packets, NULL), + BENCH_INFO(packet_last_seg, create_packets, free_packets, NULL), + BENCH_INFO(packet_next_seg, create_packets, free_packets, NULL), + BENCH_INFO(packet_seg_data, create_packets, free_packets, NULL), + BENCH_INFO(packet_seg_data_len, create_packets, free_packets, NULL), + BENCH_INFO(packet_concat, alloc_concat_packets, free_packets, NULL), + BENCH_INFO(packet_split, create_packets, free_packets_twice, NULL), + BENCH_INFO(packet_copy, create_packets, free_packets_twice, NULL), + BENCH_INFO(packet_copy_part, create_packets, free_packets_twice, NULL), + BENCH_INFO(packet_copy_to_mem, create_packets, free_packets, NULL), + BENCH_INFO(packet_copy_from_mem, create_packets, free_packets, NULL), + BENCH_INFO(packet_copy_from_pkt, alloc_packets_twice, free_packets_twice, NULL), + BENCH_INFO(packet_copy_data, create_packets, free_packets, NULL), + BENCH_INFO(packet_move_data, create_packets, free_packets, NULL), + BENCH_INFO(packet_pool, create_packets, free_packets, NULL), + BENCH_INFO(packet_input, create_packets, free_packets, NULL), + BENCH_INFO(packet_input_index, create_packets, free_packets, NULL), + BENCH_INFO(packet_user_ptr, create_packets, free_packets, NULL), + BENCH_INFO(packet_user_ptr_set, create_packets, free_packets, NULL), + BENCH_INFO(packet_user_area, create_packets, free_packets, NULL), + BENCH_INFO(packet_user_area_size, create_packets, free_packets, NULL), + BENCH_INFO(packet_l2_ptr, create_packets, free_packets, NULL), + BENCH_INFO(packet_l2_offset, create_packets, free_packets, NULL), + BENCH_INFO(packet_l2_offset_set, create_packets, free_packets, NULL), + BENCH_INFO(packet_l3_ptr, create_packets, free_packets, NULL), + BENCH_INFO(packet_l3_offset, create_packets, free_packets, NULL), + BENCH_INFO(packet_l3_offset_set, create_packets, free_packets, NULL), + BENCH_INFO(packet_l4_ptr, create_packets, free_packets, NULL), + BENCH_INFO(packet_l4_offset, create_packets, free_packets, NULL), + BENCH_INFO(packet_l4_offset_set, create_packets, free_packets, NULL), + BENCH_INFO(packet_flow_hash, create_packets, free_packets, NULL), + BENCH_INFO(packet_flow_hash_set, create_packets, free_packets, NULL), + BENCH_INFO(packet_ts, create_packets, free_packets, NULL), + BENCH_INFO(packet_ts_set, create_packets, free_packets, NULL), + BENCH_INFO(packet_ref_static, create_packets, free_packets_twice, NULL), + BENCH_INFO(packet_ref, create_packets, free_packets_twice, NULL), + BENCH_INFO(packet_ref_pkt, alloc_packets_twice, free_packets_twice, NULL), + BENCH_INFO(packet_has_ref, alloc_ref_packets, free_packets_twice, NULL), + BENCH_INFO(packet_subtype, create_packets, free_packets, NULL), + BENCH_INFO(packet_parse, alloc_parse_packets_ipv4_tcp, free_packets, + "packet_parse ipv4/tcp"), + BENCH_INFO(packet_parse, alloc_parse_packets_ipv4_udp, free_packets, + "packet_parse ipv4/udp"), + BENCH_INFO(packet_parse, alloc_parse_packets_ipv6_tcp, free_packets, + "packet_parse ipv6/tcp"), + BENCH_INFO(packet_parse, alloc_parse_packets_ipv6_udp, free_packets, + "packet_parse ipv6/udp"), + BENCH_INFO(packet_parse_multi, alloc_parse_packets_multi_ipv4_tcp, free_packets_multi, + "packet_parse_multi ipv4/tcp"), + BENCH_INFO(packet_parse_multi, alloc_parse_packets_multi_ipv4_udp, free_packets_multi, + "packet_parse_multi ipv4/udp"), + BENCH_INFO(packet_parse_multi, alloc_parse_packets_multi_ipv6_tcp, free_packets_multi, + "packet_parse_multi ipv6/tcp"), + BENCH_INFO(packet_parse_multi, alloc_parse_packets_multi_ipv6_udp, free_packets_multi, + "packet_parse_multi ipv6/udp"), }; /** @@ -1838,18 +1757,24 @@ int main(int argc, char *argv[]) capa.pkt.max_uarea_size < PKT_POOL_UAREA_SIZE) { ODPH_ERR("Error: user area size not supported.\n"); exit(EXIT_FAILURE); + } else if (gbl_args->appl.cache_size > (int)capa.pkt.max_cache_size) { + ODPH_ERR("Error: cache size not supported (max %" PRIu32 ")\n", + capa.pkt.max_cache_size); + exit(EXIT_FAILURE); } /* Create packet pool */ odp_pool_param_init(¶ms); params.pkt.seg_len = PKT_POOL_SEG_LEN; /* Using packet length as twice the TEST_MAX_PKT_SIZE as some - * test cases (bench_packet_ref_pkt) might allocate a bigger + * test cases (packet_ref_pkt) might allocate a bigger * packet than TEST_MAX_PKT_SIZE. */ params.pkt.len = 2 * TEST_MAX_PKT_SIZE; params.pkt.num = pkt_num; params.pkt.uarea_size = PKT_POOL_UAREA_SIZE; + if (gbl_args->appl.cache_size >= 0) + params.pkt.cache_size = gbl_args->appl.cache_size; params.type = ODP_POOL_PACKET; gbl_args->pool = odp_pool_create("packet pool", ¶ms); @@ -1863,6 +1788,10 @@ int main(int argc, char *argv[]) printf("CPU mask: %s\n", cpumaskstr); printf("Burst size: %d\n", gbl_args->appl.burst_size); printf("Bench repeat: %d\n", TEST_REPEAT_COUNT); + if (gbl_args->appl.cache_size < 0) + printf("Pool cache size: default\n"); + else + printf("Pool cache size: %d\n", gbl_args->appl.cache_size); odp_pool_print(gbl_args->pool); diff --git a/test/performance/odp_crypto.c b/test/performance/odp_crypto.c index 46eb7141e..94d9cc6cf 100644 --- a/test/performance/odp_crypto.c +++ b/test/performance/odp_crypto.c @@ -26,6 +26,7 @@ #define POOL_NUM_PKT 64 #define AAD_LEN 8 /* typical AAD length used in IPsec when ESN is not in use */ +#define MAX_AUTH_DIGEST_LEN 32 /* maximum MAC length in bytes */ static uint8_t test_aad[AAD_LEN] = "01234567"; static uint8_t test_iv[16] = "0123456789abcdef"; @@ -407,6 +408,49 @@ static crypto_alg_config_t algs_config[] = { .auth_aad_len = AAD_LEN, }, }, + { + .name = "zuc-eea3", + .session = { + .cipher_alg = ODP_CIPHER_ALG_ZUC_EEA3, + .cipher_key = { + .data = test_key16, + .length = sizeof(test_key16) + }, + .cipher_iv_len = 16, + .auth_alg = ODP_AUTH_ALG_NULL, + }, + }, + { + .name = "zuc-eia3", + .session = { + .cipher_alg = ODP_CIPHER_ALG_NULL, + .auth_alg = ODP_AUTH_ALG_ZUC_EIA3, + .auth_key = { + .data = test_key16, + .length = sizeof(test_key16) + }, + .auth_iv_len = 16, + .auth_digest_len = 4, + }, + }, + { + .name = "zuc-eea3-zuc-eia3", + .session = { + .cipher_alg = ODP_CIPHER_ALG_ZUC_EEA3, + .cipher_key = { + .data = test_key16, + .length = sizeof(test_key16) + }, + .cipher_iv_len = 16, + .auth_alg = ODP_AUTH_ALG_ZUC_EIA3, + .auth_key = { + .data = test_key16, + .length = sizeof(test_key16) + }, + .auth_iv_len = 16, + .auth_digest_len = 4, + }, + }, }; /** @@ -516,7 +560,7 @@ get_elapsed_usec(time_record_t *start, time_record_t *end) return e - s; } -#define REPORT_HEADER "\n%30.30s %15s %15s %15s %15s %15s %15s\n" +#define REPORT_HEADER "%30.30s %15s %15s %15s %15s %15s %15s\n" #define REPORT_LINE "%30.30s %15d %15d %15.3f %15.3f %15.3f %15d\n" /** @@ -610,6 +654,7 @@ create_session_from_config(odp_crypto_session_t *session, odp_crypto_session_param_init(¶ms); memcpy(¶ms, &config->session, sizeof(odp_crypto_session_param_t)); params.op = ODP_CRYPTO_OP_ENCODE; + params.auth_cipher_text = true; /* Lookup the packet pool */ pkt_pool = odp_pool_lookup("packet_pool"); @@ -633,6 +678,19 @@ create_session_from_config(odp_crypto_session_t *session, } if (odp_crypto_session_create(¶ms, session, &ses_create_rc)) { + switch (ses_create_rc) { + case ODP_CRYPTO_SES_ERR_ALG_COMBO: + printf(" requested algorithm combination not supported\n"); + return 1; + case ODP_CRYPTO_SES_ERR_ALG_ORDER: + printf(" requested algorithm order not supported\n"); + return 1; + case ODP_CRYPTO_SES_ERR_PARAMS: + printf(" requested session parameters not supported\n"); + return 1; + default: + break; + } ODPH_ERR("crypto session create failed.\n"); return -1; } @@ -675,6 +733,7 @@ run_measure_one(crypto_args_t *cargs, odp_queue_t out_queue; odp_packet_t pkt = ODP_PACKET_INVALID; int rc = 0; + uint32_t packet_len = payload_length + MAX_AUTH_DIGEST_LEN; pkt_pool = odp_pool_lookup("packet_pool"); if (pkt_pool == ODP_POOL_INVALID) { @@ -691,7 +750,7 @@ run_measure_one(crypto_args_t *cargs, } if (cargs->reuse_packet) { - pkt = make_packet(pkt_pool, payload_length); + pkt = make_packet(pkt_pool, packet_len); if (ODP_PACKET_INVALID == pkt) return -1; } @@ -727,7 +786,7 @@ run_measure_one(crypto_args_t *cargs, odp_packet_t out_pkt; if (!cargs->reuse_packet) { - pkt = make_packet(pkt_pool, payload_length); + pkt = make_packet(pkt_pool, packet_len); if (ODP_PACKET_INVALID == pkt) return -1; } @@ -874,6 +933,9 @@ static int check_cipher_alg(const odp_crypto_capability_t *capa, if (capa->ciphers.bit.chacha20_poly1305) return 0; break; + case ODP_CIPHER_ALG_ZUC_EEA3: + if (capa->ciphers.bit.zuc_eea3) + return 0; default: break; } @@ -925,6 +987,9 @@ static int check_auth_alg(const odp_crypto_capability_t *capa, if (capa->auths.bit.chacha20_poly1305) return 0; break; + case ODP_AUTH_ALG_ZUC_EIA3: + if (capa->auths.bit.zuc_eia3) + return 0; default: break; } @@ -983,6 +1048,11 @@ static int check_auth_params(const odp_crypto_capability_t *crypto_capa, { int num, rc; + if (param->auth_digest_len > MAX_AUTH_DIGEST_LEN) { + ODPH_ERR("MAX_AUTH_DIGEST_LEN too low\n"); + return 1; + } + if (check_auth_alg(crypto_capa, param->auth_alg)) return 1; @@ -1026,6 +1096,8 @@ static int run_measure_one_config(test_run_arg_t *arg) odp_crypto_capability_t crypto_capa = arg->crypto_capa; int rc = 0; + printf("\n"); + if (check_cipher_params(&crypto_capa, &config->session, &config->cipher_in_bit_mode)) { printf(" Cipher algorithm not supported\n"); @@ -1038,14 +1110,13 @@ static int run_measure_one_config(test_run_arg_t *arg) rc = 1; } + if (rc == 0) + rc = create_session_from_config(&session, config, cargs); if (rc) { - printf(" => %s skipped\n\n", config->name); - return 0; + printf(" => %s skipped\n", config->name); + return rc > 0 ? 0 : -1; } - if (create_session_from_config(&session, config, cargs)) - return -1; - if (cargs->payload_length) { rc = run_measure_one(cargs, config, &session, cargs->payload_length, &result); @@ -1150,7 +1221,7 @@ int main(int argc, char *argv[]) max_seg_len = pool_capa.pkt.max_seg_len; for (i = 0; i < sizeof(payloads) / sizeof(unsigned int); i++) { - if (payloads[i] > max_seg_len) + if (payloads[i] + MAX_AUTH_DIGEST_LEN > max_seg_len) break; } diff --git a/test/performance/odp_dma_perf.c b/test/performance/odp_dma_perf.c index 26397bf49..615107299 100644 --- a/test/performance/odp_dma_perf.c +++ b/test/performance/odp_dma_perf.c @@ -110,16 +110,16 @@ static void set_option_defaults(test_config_t *config) static void parse_completion_modes(test_config_t *config, const char *optarg) { char *tmp_str = strdup(optarg); - char *tmp = strtok(tmp_str, COMPL_DELIMITER); + char *tmp; int mode; uint32_t i = 0U; config->compl_modes.num_modes = 0; - if (tmp == NULL) { - free(tmp_str); + if (tmp_str == NULL) return; - } + + tmp = strtok(tmp_str, COMPL_DELIMITER); while (tmp) { mode = atoi(tmp); diff --git a/test/performance/odp_packet_gen.c b/test/performance/odp_packet_gen.c index 1f74ad006..e02ffe95e 100644 --- a/test/performance/odp_packet_gen.c +++ b/test/performance/odp_packet_gen.c @@ -36,6 +36,9 @@ /* Minimum number of packets to receive in CI test */ #define MIN_RX_PACKETS_CI 800 +/* Identifier for payload-timestamped packets */ +#define TS_MAGIC 0xff88ee99ddaaccbb + ODP_STATIC_ASSERT(MAX_PKTIOS <= UINT8_MAX, "Interface index must fit into uint8_t\n"); typedef struct test_options_t { @@ -49,6 +52,7 @@ typedef struct test_options_t { uint32_t num_pkt; uint32_t pkt_len; uint8_t use_rand_pkt_len; + uint8_t direct_rx; uint32_t rand_pkt_len_min; uint32_t rand_pkt_len_max; uint32_t rand_pkt_len_bins; @@ -63,7 +67,11 @@ typedef struct test_options_t { uint32_t wait_sec; uint32_t wait_start_sec; uint32_t mtu; + odp_bool_t use_refs; odp_bool_t promisc_mode; + odp_bool_t calc_latency; + odp_bool_t calc_cs; + odp_bool_t fill_pl; struct vlan_hdr { uint16_t tpid; @@ -88,6 +96,9 @@ typedef struct thread_arg_t { /* pktout queue per pktio interface (per thread) */ odp_pktout_queue_t pktout[MAX_PKTIOS]; + /* In direct_rx mode, pktin queue per pktio interface (per thread) */ + odp_pktin_queue_t pktin[MAX_PKTIOS]; + } thread_arg_t; typedef struct ODP_ALIGNED_CACHE thread_stat_t { @@ -95,6 +106,10 @@ typedef struct ODP_ALIGNED_CACHE thread_stat_t { uint64_t rx_timeouts; uint64_t rx_packets; uint64_t rx_bytes; + uint64_t rx_lat_nsec; + uint64_t rx_lat_min_nsec; + uint64_t rx_lat_max_nsec; + uint64_t rx_lat_packets; uint64_t tx_timeouts; uint64_t tx_packets; @@ -127,6 +142,7 @@ typedef struct test_global_t { odph_ethaddr_t eth_dst; odp_pktio_t pktio; odp_pktout_queue_t pktout[ODP_THREAD_COUNT_MAX]; + odp_pktin_queue_t pktin[ODP_THREAD_COUNT_MAX]; int started; } pktio[MAX_PKTIOS]; @@ -136,6 +152,18 @@ typedef struct test_global_t { } test_global_t; +typedef struct ODP_PACKED { + uint64_t magic; + uint64_t tx_ts; +} ts_data_t; + +typedef struct { + uint64_t nsec; + uint64_t min; + uint64_t max; + uint64_t packets; +} rx_lat_data_t; + static test_global_t *test_global; static void print_usage(void) @@ -149,8 +177,8 @@ static void print_usage(void) " -i, --interface <name> Packet IO interfaces. Comma-separated list of\n" " interface names (no spaces) e.g. eth0,eth1.\n" " At least one interface is required.\n" - "\n" - " Optional:\n" + "\n"); + printf(" Optional:\n" " -e, --eth_dst <mac> Destination MAC address. Comma-separated list of\n" " addresses (no spaces), one address per packet IO\n" " interface e.g. AA:BB:CC:DD:EE:FF,11:22:33:44:55:66\n" @@ -167,11 +195,18 @@ static void print_usage(void) " -L, --len_range <min,max,bins>\n" " Random packet length. Specify the minimum and maximum\n" " packet lengths and the number of bins. To reduce pool size\n" - " requirement the length range can be divined into even sized\n" + " requirement the length range can be divided into even sized\n" " bins. Min and max size packets are always used and included\n" " into the number of bins (bins >= 2). Bin value of 0 means\n" " that each packet length is used. Comma-separated (no spaces).\n" " Overrides standard packet length option.\n" + " -D, --direct_rx Direct input mode (default: 0)\n" + " 0: Use scheduler for packet input\n" + " 1: Poll packet input in direct mode\n"); + printf(" -R, --no_pkt_refs Do not use packet references. Always allocate a\n" + " fresh set of packets for a transmit burst. Some\n" + " features may be available only with references\n" + " disabled.\n" " -M, --mtu <len> Interface MTU in bytes.\n" " -b, --burst_size Transmit burst size. Default: 8\n" " -x, --bursts Number of bursts per one transmit round. Default: 1\n" @@ -183,11 +218,18 @@ static void print_usage(void) " -o, --udp_src UDP source port. Default: 10000\n" " -p, --udp_dst UDP destination port. Default: 20000\n" " -P, --promisc_mode Enable promiscuous mode.\n" + " -a, --latency Calculate latency. Disables packet references (see\n" + " \"--no_pkt_refs\").\n" " -c, --c_mode <counts> Counter mode for incrementing UDP port numbers.\n" " Specify the number of port numbers used starting from\n" " udp_src/udp_dst. Comma-separated (no spaces) list of\n" " count values: <udp_src count>,<udp_dst count>\n" " Default value: 0,0\n" + " -C, --no_udp_checksum Do not calculate UDP checksum. Instead, set it to\n" + " zero in every packet.\n" + " -A, --no_payload_fill Do not fill payload. By default, payload is filled\n" + " with a pattern until the end of first packet\n" + " segment.\n" " -q, --quit Quit after this many transmit rounds.\n" " Default: 0 (don't quit)\n" " -u, --update_stat <msec> Update and print statistics every <msec> milliseconds.\n" @@ -256,7 +298,9 @@ static int parse_options(int argc, char *argv[], test_global_t *global) {"num_tx", required_argument, NULL, 't'}, {"num_pkt", required_argument, NULL, 'n'}, {"len", required_argument, NULL, 'l'}, - {"len_range", required_argument, NULL, 'L'}, + {"len_range", required_argument, NULL, 'L'}, + {"direct_rx", required_argument, NULL, 'D'}, + {"no_pkt_refs", no_argument, NULL, 'R'}, {"burst_size", required_argument, NULL, 'b'}, {"bursts", required_argument, NULL, 'x'}, {"gap", required_argument, NULL, 'g'}, @@ -265,8 +309,11 @@ static int parse_options(int argc, char *argv[], test_global_t *global) {"ipv4_dst", required_argument, NULL, 'd'}, {"udp_src", required_argument, NULL, 'o'}, {"udp_dst", required_argument, NULL, 'p'}, - {"promisc_mode", no_argument, NULL, 'P'}, + {"promisc_mode", no_argument, NULL, 'P'}, + {"latency", no_argument, NULL, 'a'}, {"c_mode", required_argument, NULL, 'c'}, + {"no_udp_checksum", no_argument, NULL, 'C'}, + {"no_payload_fill", no_argument, NULL, 'A'}, {"mtu", required_argument, NULL, 'M'}, {"quit", required_argument, NULL, 'q'}, {"wait", required_argument, NULL, 'w'}, @@ -276,7 +323,7 @@ static int parse_options(int argc, char *argv[], test_global_t *global) {NULL, 0, NULL, 0} }; - static const char *shortopts = "+i:e:r:t:n:l:L:M:b:x:g:v:s:d:o:p:c:q:u:w:W:Ph"; + static const char *shortopts = "+i:e:r:t:n:l:L:D:RM:b:x:g:v:s:d:o:p:c:CAq:u:w:W:Pah"; test_options->num_pktio = 0; test_options->num_rx = 1; @@ -284,11 +331,16 @@ static int parse_options(int argc, char *argv[], test_global_t *global) test_options->num_pkt = 1000; test_options->pkt_len = 512; test_options->use_rand_pkt_len = 0; + test_options->direct_rx = 0; + test_options->use_refs = 1; test_options->burst_size = 8; test_options->bursts = 1; test_options->gap_nsec = 1000000; test_options->num_vlan = 0; test_options->promisc_mode = 0; + test_options->calc_latency = 0; + test_options->calc_cs = 1; + test_options->fill_pl = 1; strncpy(test_options->ipv4_src_s, "192.168.0.1", sizeof(test_options->ipv4_src_s) - 1); strncpy(test_options->ipv4_dst_s, "192.168.0.2", @@ -333,14 +385,13 @@ static int parse_options(int argc, char *argv[], test_global_t *global) str_len -= len + 1; if (i == MAX_PKTIOS) { - printf("Error: Too many interfaces\n"); + ODPH_ERR("Error: Too many interfaces\n"); ret = -1; break; } if (len > MAX_PKTIO_NAME) { - printf("Error: Too long interface name %s\n", - str); + ODPH_ERR("Error: Too long interface name %s\n", str); ret = -1; break; } @@ -366,14 +417,13 @@ static int parse_options(int argc, char *argv[], test_global_t *global) str_len -= len + 1; if (i == MAX_PKTIOS) { - printf("Error: Too many MAC addresses\n"); + ODPH_ERR("Error: Too many MAC addresses\n"); ret = -1; break; } if (odph_eth_addr_parse(dst, str)) { - printf("Error: Bad MAC address: %s\n", - str); + ODPH_ERR("Error: Bad MAC address: %s\n", str); ret = -1; break; } @@ -385,7 +435,7 @@ static int parse_options(int argc, char *argv[], test_global_t *global) case 'o': udp_port = atoi(optarg); if (udp_port < 0 || udp_port > UINT16_MAX) { - printf("Error: Bad UDP source port: %d\n", udp_port); + ODPH_ERR("Error: Bad UDP source port: %d\n", udp_port); ret = -1; break; } @@ -394,7 +444,7 @@ static int parse_options(int argc, char *argv[], test_global_t *global) case 'p': udp_port = atoi(optarg); if (udp_port < 0 || udp_port > UINT16_MAX) { - printf("Error: Bad UDP destination port: %d\n", udp_port); + ODPH_ERR("Error: Bad UDP destination port: %d\n", udp_port); ret = -1; break; } @@ -403,6 +453,9 @@ static int parse_options(int argc, char *argv[], test_global_t *global) case 'P': test_options->promisc_mode = 1; break; + case 'a': + test_options->calc_latency = 1; + break; case 'r': test_options->num_rx = atoi(optarg); break; @@ -426,6 +479,12 @@ static int parse_options(int argc, char *argv[], test_global_t *global) test_options->rand_pkt_len_bins = val; test_options->use_rand_pkt_len = 1; break; + case 'D': + test_options->direct_rx = atoi(optarg); + break; + case 'R': + test_options->use_refs = 0; + break; case 'M': test_options->mtu = atoi(optarg); break; @@ -441,15 +500,14 @@ static int parse_options(int argc, char *argv[], test_global_t *global) case 'v': test_options->num_vlan = parse_vlan(optarg, global); if (test_options->num_vlan == 0) { - printf("Error: Did not find any VLANs\n"); + ODPH_ERR("Error: Did not find any VLANs\n"); ret = -1; } break; case 's': if (odph_ipv4_addr_parse(&test_options->ipv4_src, optarg)) { - printf("Error: Bad IPv4 source address: %s\n", - optarg); + ODPH_ERR("Error: Bad IPv4 source address: %s\n", optarg); ret = -1; } strncpy(test_options->ipv4_src_s, optarg, @@ -458,8 +516,7 @@ static int parse_options(int argc, char *argv[], test_global_t *global) case 'd': if (odph_ipv4_addr_parse(&test_options->ipv4_dst, optarg)) { - printf("Error: Bad IPv4 destination address: %s\n", - optarg); + ODPH_ERR("Error: Bad IPv4 destination address: %s\n", optarg); ret = -1; } strncpy(test_options->ipv4_dst_s, optarg, @@ -473,6 +530,12 @@ static int parse_options(int argc, char *argv[], test_global_t *global) count = strtoul(end, NULL, 0); test_options->c_mode.udp_dst = count; break; + case 'C': + test_options->calc_cs = 0; + break; + case 'A': + test_options->fill_pl = 0; + break; case 'q': test_options->quit = atoll(optarg); break; @@ -496,13 +559,13 @@ static int parse_options(int argc, char *argv[], test_global_t *global) } if (help == 0 && test_options->num_pktio == 0) { - printf("Error: At least one packet IO interface is needed.\n"); - printf(" Use -i <name> to specify interfaces.\n"); + ODPH_ERR("Error: At least one packet IO interface is needed.\n"); + ODPH_ERR(" Use -i <name> to specify interfaces.\n"); ret = -1; } if (test_options->num_rx < 1 || test_options->num_tx < 1) { - printf("Error: At least one rx and tx thread needed.\n"); + ODPH_ERR("Error: At least one rx and tx thread needed.\n"); ret = -1; } @@ -516,21 +579,21 @@ static int parse_options(int argc, char *argv[], test_global_t *global) uint32_t req_pkts; if (test_options->rand_pkt_len_max <= test_options->rand_pkt_len_min) { - printf("Error: Bad max packet length\n"); + ODPH_ERR("Error: Bad max packet length\n"); ret = -1; } if (pkt_bins == 1) { - printf("Error: Invalid bins value\n"); + ODPH_ERR("Error: Invalid bins value\n"); ret = -1; } if (pkt_sizes < pkt_bins) { - printf("Error: Not enough packet sizes for %" PRIu32 " bins\n", pkt_bins); + ODPH_ERR("Error: Not enough packet sizes for %" PRIu32 " bins\n", pkt_bins); ret = -1; } if (pkt_bins && num_tx_pkt > pkt_bins && num_tx_pkt % pkt_bins) - printf("\nWARNING: Transmit packet count is not evenly divisible into packet length bins.\n\n"); + ODPH_ERR("\nWARNING: Transmit packet count is not evenly divisible into packet length bins.\n\n"); else if (!pkt_bins && num_tx_pkt > pkt_sizes && num_tx_pkt % pkt_sizes) - printf("\nWARNING: Transmit packet count is not evenly divisible into packet lengths.\n\n"); + ODPH_ERR("\nWARNING: Transmit packet count is not evenly divisible into packet lengths.\n\n"); req_pkts = pkt_bins ? pkt_bins : pkt_sizes; if (req_pkts > num_tx_pkt) @@ -545,27 +608,29 @@ static int parse_options(int argc, char *argv[], test_global_t *global) test_options->burst_size); if (test_options->num_pkt < min_packets) { - printf("Error: Pool needs to have at least %u packets\n", - min_packets); + ODPH_ERR("Error: Pool needs to have at least %u packets\n", min_packets); ret = -1; } + if (test_options->calc_latency) + test_options->use_refs = 0; + if (test_options->gap_nsec) { double gap_hz = 1000000000.0 / test_options->gap_nsec; if (gap_hz > (double)odp_time_local_res()) { - printf("\nWARNING: Burst gap exceeds time counter resolution " - "%" PRIu64 "\n\n", odp_time_local_res()); + ODPH_ERR("\nWARNING: Burst gap exceeds time counter resolution " + "%" PRIu64 "\n\n", odp_time_local_res()); } } if (test_options->c_mode.udp_dst && num_tx_pkt % test_options->c_mode.udp_dst) - printf("\nWARNING: Transmit packet count is not evenly divisible by UDP destination port count.\n\n"); + ODPH_ERR("\nWARNING: Transmit packet count is not evenly divisible by UDP destination port count.\n\n"); if (test_options->c_mode.udp_src && num_tx_pkt % test_options->c_mode.udp_src) - printf("\nWARNING: Transmit packet count is not evenly divisible by UDP source port count.\n\n"); + ODPH_ERR("\nWARNING: Transmit packet count is not evenly divisible by UDP source port count.\n\n"); test_options->hdr_len = ODPH_ETHHDR_LEN + (test_options->num_vlan * ODPH_VLANHDR_LEN) + @@ -574,7 +639,7 @@ static int parse_options(int argc, char *argv[], test_global_t *global) pkt_len = test_options->use_rand_pkt_len ? test_options->rand_pkt_len_min : test_options->pkt_len; if (test_options->hdr_len >= pkt_len) { - printf("Error: Headers do not fit into packet length %" PRIu32 "\n", pkt_len); + ODPH_ERR("Error: Headers do not fit into packet length %" PRIu32 "\n", pkt_len); ret = -1; } @@ -589,8 +654,8 @@ static int set_num_cpu(test_global_t *global) /* One thread used for the main thread */ if (num_cpu > ODP_THREAD_COUNT_MAX - 1) { - printf("Error: Too many threads. API supports max %i.\n", - ODP_THREAD_COUNT_MAX - 1); + ODPH_ERR("Error: Too many threads. API supports max %i.\n", + ODP_THREAD_COUNT_MAX - 1); return -1; } @@ -601,8 +666,7 @@ static int set_num_cpu(test_global_t *global) /* Normally we want to use only worker threads */ if (ret > 1) { - printf("Error: Too many workers. Maximum supported %i.\n", - ret); + ODPH_ERR("Error: Too many workers. Maximum supported %i.\n", ret); return -1; } @@ -610,8 +674,7 @@ static int set_num_cpu(test_global_t *global) * we try to use any CPUs available. */ ret = odp_cpumask_all_available(&global->cpumask); if (ret < num_cpu) { - printf("Error: Not enough CPUs. Maximum supported %i.\n", - ret); + ODPH_ERR("Error: Not enough CPUs. Maximum supported %i.\n", ret); return -1; } @@ -644,18 +707,19 @@ static int open_pktios(test_global_t *global) uint32_t i, seg_len; int j, pktio_idx; test_options_t *test_options = &global->test_options; - uint32_t num_rx = test_options->num_rx; + int num_rx = test_options->num_rx; int num_tx = test_options->num_tx; uint32_t num_pktio = test_options->num_pktio; uint32_t num_pkt = test_options->num_pkt; uint32_t pkt_len = test_options->use_rand_pkt_len ? test_options->rand_pkt_len_max : test_options->pkt_len; odp_pktout_queue_t pktout[num_tx]; + odp_pktin_queue_t pktin[num_rx]; printf("\nODP packet generator\n"); printf(" quit test after %" PRIu64 " rounds\n", test_options->quit); - printf(" num rx threads %u\n", num_rx); + printf(" num rx threads %i\n", num_rx); printf(" num tx threads %i\n", num_tx); printf(" num packets %u\n", num_pkt); if (test_options->use_rand_pkt_len) @@ -670,7 +734,12 @@ static int open_pktios(test_global_t *global) printf("%u bytes\n", test_options->mtu); else printf("interface default\n"); + printf(" packet input mode: %s\n", test_options->direct_rx ? "direct" : "scheduler"); printf(" promisc mode: %s\n", test_options->promisc_mode ? "enabled" : "disabled"); + printf(" packet references: %s\n", test_options->use_refs ? "enabled" : "disabled"); + printf(" measure latency: %s\n", test_options->calc_latency ? "enabled" : "disabled"); + printf(" UDP checksum: %s\n", test_options->calc_cs ? "enabled" : "disabled"); + printf(" payload filling: %s\n", test_options->fill_pl ? "enabled" : "disabled"); printf(" tx burst size %u\n", test_options->burst_size); printf(" tx bursts %u\n", test_options->bursts); printf(" tx burst gap %" PRIu64 " nsec\n", @@ -710,28 +779,26 @@ static int open_pktios(test_global_t *global) global->pool = ODP_POOL_INVALID; if (odp_pool_capability(&pool_capa)) { - printf("Error: Pool capability failed.\n"); + ODPH_ERR("Error: Pool capability failed.\n"); return -1; } if (pool_capa.pkt.max_num && num_pkt > pool_capa.pkt.max_num) { - printf("Error: Too many packets. Max %u supported.\n", - pool_capa.pkt.max_num); + ODPH_ERR("Error: Too many packets. Max %u supported.\n", pool_capa.pkt.max_num); return -1; } if (pool_capa.pkt.max_len && pkt_len > pool_capa.pkt.max_len) { - printf("Error: Too large packets. Max %u supported length.\n", - pool_capa.pkt.max_len); + ODPH_ERR("Error: Too large packets. Max %u supported length.\n", + pool_capa.pkt.max_len); return -1; } seg_len = test_options->hdr_len; if (pool_capa.pkt.max_seg_len && seg_len > pool_capa.pkt.max_seg_len) { - printf("Error: Max segment length is too small %u\n", - pool_capa.pkt.max_seg_len); + ODPH_ERR("Error: Max segment length is too small %u\n", pool_capa.pkt.max_seg_len); return -1; } @@ -745,17 +812,22 @@ static int open_pktios(test_global_t *global) pool = odp_pool_create("packet gen pool", &pool_param); if (pool == ODP_POOL_INVALID) { - printf("Error: Pool create failed.\n"); + ODPH_ERR("Error: Pool create failed.\n"); return -1; } global->pool = pool; if (odp_pktio_max_index() >= MAX_PKTIO_INDEXES) - printf("Warning: max pktio index (%u) is too large\n", odp_pktio_max_index()); + ODPH_ERR("Warning: max pktio index (%u) is too large\n", odp_pktio_max_index()); odp_pktio_param_init(&pktio_param); - pktio_param.in_mode = ODP_PKTIN_MODE_SCHED; + + if (test_options->direct_rx) + pktio_param.in_mode = ODP_PKTIN_MODE_DIRECT; + else + pktio_param.in_mode = ODP_PKTIN_MODE_SCHED; + pktio_param.out_mode = ODP_PKTOUT_MODE_DIRECT; for (i = 0; i < num_pktio; i++) @@ -767,7 +839,7 @@ static int open_pktios(test_global_t *global) pktio = odp_pktio_open(name, pool, &pktio_param); if (pktio == ODP_PKTIO_INVALID) { - printf("Error (%s): Pktio open failed.\n", name); + ODPH_ERR("Error (%s): Pktio open failed.\n", name); return -1; } @@ -777,32 +849,32 @@ static int open_pktios(test_global_t *global) pktio_idx = odp_pktio_index(pktio); if (pktio_idx < 0 || pktio_idx >= MAX_PKTIO_INDEXES) { - printf("Error (%s): Bad pktio index: %i\n", name, pktio_idx); + ODPH_ERR("Error (%s): Bad pktio index: %i\n", name, pktio_idx); return -1; } global->if_from_pktio_idx[pktio_idx] = i; if (odp_pktio_capability(pktio, &pktio_capa)) { - printf("Error (%s): Pktio capability failed.\n", name); + ODPH_ERR("Error (%s): Pktio capability failed.\n", name); return -1; } - if (num_rx > pktio_capa.max_input_queues) { - printf("Error (%s): Too many RX threads. Interface supports max %u input queues.\n", - name, pktio_capa.max_input_queues); + if (num_rx > (int)pktio_capa.max_input_queues) { + ODPH_ERR("Error (%s): Too many RX threads. Interface supports max %u input queues.\n", + name, pktio_capa.max_input_queues); return -1; } if (num_tx > (int)pktio_capa.max_output_queues) { - printf("Error (%s): Too many TX threads. Interface supports max %u output queues.\n", - name, pktio_capa.max_output_queues); + ODPH_ERR("Error (%s): Too many TX threads. Interface supports max %u output queues.\n", + name, pktio_capa.max_output_queues); return -1; } if (odp_pktio_mac_addr(pktio, &global->pktio[i].eth_src.addr, ODPH_ETHADDR_LEN) != ODPH_ETHADDR_LEN) { - printf("Error (%s): MAC address read failed.\n", name); + ODPH_ERR("Error (%s): MAC address read failed.\n", name); return -1; } @@ -860,15 +932,23 @@ static int open_pktios(test_global_t *global) odp_pktin_queue_param_init(&pktin_param); - pktin_param.queue_param.sched.prio = odp_schedule_default_prio(); - pktin_param.queue_param.sched.sync = ODP_SCHED_SYNC_PARALLEL; - pktin_param.queue_param.sched.group = ODP_SCHED_GROUP_ALL; - pktin_param.hash_enable = 1; - pktin_param.hash_proto.proto.ipv4_udp = 1; + if (test_options->direct_rx) { + pktin_param.op_mode = ODP_PKTIO_OP_MT_UNSAFE; + } else { + pktin_param.queue_param.sched.prio = odp_schedule_default_prio(); + pktin_param.queue_param.sched.sync = ODP_SCHED_SYNC_PARALLEL; + pktin_param.queue_param.sched.group = ODP_SCHED_GROUP_ALL; + } + pktin_param.num_queues = num_rx; + if (num_rx > 1) { + pktin_param.hash_enable = 1; + pktin_param.hash_proto.proto.ipv4_udp = 1; + } + if (odp_pktin_queue_config(pktio, &pktin_param)) { - printf("Error (%s): Pktin config failed.\n", name); + ODPH_ERR("Error (%s): Pktin config failed.\n", name); return -1; } @@ -877,18 +957,27 @@ static int open_pktios(test_global_t *global) pktout_param.num_queues = num_tx; if (odp_pktout_queue_config(pktio, &pktout_param)) { - printf("Error (%s): Pktout config failed.\n", name); + ODPH_ERR("Error (%s): Pktout config failed.\n", name); return -1; } if (odp_pktout_queue(pktio, pktout, num_tx) != num_tx) { - printf("Error (%s): Pktout queue request failed.\n", - name); + ODPH_ERR("Error (%s): Pktout queue request failed.\n", name); return -1; } for (j = 0; j < num_tx; j++) global->pktio[i].pktout[j] = pktout[j]; + + if (test_options->direct_rx) { + if (odp_pktin_queue(pktio, pktin, num_rx) != num_rx) { + ODPH_ERR("Error (%s): Pktin queue request failed.\n", name); + return -1; + } + + for (j = 0; j < num_rx; j++) + global->pktio[i].pktin[j] = pktin[j]; + } } return 0; @@ -899,7 +988,7 @@ static int print_link_info(odp_pktio_t pktio) odp_pktio_link_info_t info; if (odp_pktio_link_info(pktio, &info)) { - printf("Error: Pktio link info failed.\n"); + ODPH_ERR("Error: Pktio link info failed.\n"); return -1; } @@ -929,8 +1018,7 @@ static int start_pktios(test_global_t *global) for (i = 0; i < num_pktio; i++) { if (odp_pktio_start(global->pktio[i].pktio)) { - printf("Error (%s): Pktio start failed.\n", - test_options->pktio_name[i]); + ODPH_ERR("Error (%s): Pktio start failed.\n", test_options->pktio_name[i]); return -1; } @@ -946,16 +1034,16 @@ static int start_pktios(test_global_t *global) if (odp_pktio_link_status(pktio) == ODP_PKTIO_LINK_STATUS_UP) { printf("pktio:%s\n", test_options->pktio_name[i]); if (print_link_info(pktio)) { - printf("Error (%s): Printing link info failed.\n", - test_options->pktio_name[i]); + ODPH_ERR("Error (%s): Printing link info failed.\n", + test_options->pktio_name[i]); return -1; } break; } link_wait++; if (link_wait > test_options->wait_sec) { - printf("Error (%s): Pktio link down.\n", - test_options->pktio_name[i]); + ODPH_ERR("Error (%s): Pktio link down.\n", + test_options->pktio_name[i]); return -1; } odp_time_wait_ns(ODP_TIME_SEC_IN_NS); @@ -983,8 +1071,7 @@ static int stop_pktios(test_global_t *global) continue; if (odp_pktio_stop(pktio)) { - printf("Error (%s): Pktio stop failed.\n", - test_options->pktio_name[i]); + ODPH_ERR("Error (%s): Pktio stop failed.\n", test_options->pktio_name[i]); ret = -1; } } @@ -1007,30 +1094,51 @@ static int close_pktios(test_global_t *global) continue; if (odp_pktio_close(pktio)) { - printf("Error (%s): Pktio close failed.\n", - test_options->pktio_name[i]); + ODPH_ERR("Error (%s): Pktio close failed.\n", test_options->pktio_name[i]); ret = -1; } } if (global->pool != ODP_POOL_INVALID && odp_pool_destroy(global->pool)) { - printf("Error: Pool destroy failed.\n"); + ODPH_ERR("Error: Pool destroy failed.\n"); ret = -1; } return ret; } +static inline void get_timestamp(odp_packet_t pkt, uint32_t ts_off, rx_lat_data_t *lat_data, + uint64_t rx_ts) +{ + ts_data_t ts_data; + uint64_t nsec; + + if (odp_unlikely(odp_packet_copy_to_mem(pkt, ts_off, sizeof(ts_data), &ts_data) < 0 || + ts_data.magic != TS_MAGIC)) + return; + + nsec = rx_ts - ts_data.tx_ts; + + if (nsec < lat_data->min) + lat_data->min = nsec; + + if (nsec > lat_data->max) + lat_data->max = nsec; + + lat_data->nsec += nsec; + lat_data->packets++; +} + static int rx_thread(void *arg) { int i, thr, num; uint32_t exit_test; uint64_t bytes; odp_time_t t1, t2, exit_time; - odp_packet_t pkt; thread_arg_t *thread_arg = arg; test_global_t *global = thread_arg->global; + int direct_rx = global->test_options.direct_rx; int periodic_stat = global->test_options.update_msec ? 1 : 0; uint64_t rx_timeouts = 0; uint64_t rx_packets = 0; @@ -1040,17 +1148,51 @@ static int rx_thread(void *arg) int clock_started = 0; int exit_timer_started = 0; int paused = 0; - int max_num = 32; - odp_event_t ev[max_num]; + const int max_num = 32; + int pktin = 0; + int num_pktio = global->test_options.num_pktio; + odp_pktin_queue_t pktin_queue[num_pktio]; + odp_packet_t pkt[max_num]; + uint32_t ts_off = global->test_options.calc_latency ? global->test_options.hdr_len : 0; + uint64_t rx_ts = 0; + rx_lat_data_t rx_lat_data = { .nsec = 0, .min = UINT64_MAX, .max = 0, .packets = 0 }; thr = odp_thread_id(); global->stat[thr].thread_type = RX_THREAD; + if (direct_rx) { + for (i = 0; i < num_pktio; i++) + pktin_queue[i] = thread_arg->pktin[i]; + } + /* Start all workers at the same time */ odp_barrier_wait(&global->barrier); while (1) { - num = odp_schedule_multi_no_wait(NULL, ev, max_num); + if (direct_rx) { + num = odp_pktin_recv(pktin_queue[pktin], pkt, max_num); + + if (odp_unlikely(num < 0)) { + ODPH_ERR("pktin (%i) recv failed: %i\n", pktin, num); + ret = -1; + num = 0; + break; + } + + pktin++; + if (pktin >= num_pktio) + pktin = 0; + } else { + odp_event_t ev[max_num]; + + num = odp_schedule_multi_no_wait(NULL, ev, max_num); + + if (num) + odp_packet_from_event_multi(pkt, ev, num); + } + + if (ts_off && num) + rx_ts = odp_time_global_ns(); exit_test = odp_atomic_load_u32(&global->exit_test); if (exit_test) { @@ -1061,11 +1203,11 @@ static int rx_thread(void *arg) exit_timer_started = 1; } else if (odp_time_diff_ns(odp_time_local(), exit_time) > ODP_TIME_SEC_IN_NS) { - if (paused == 0) { + if (direct_rx == 0 && paused == 0) { odp_schedule_pause(); paused = 1; } else if (num == 0) { - /* Exit schedule loop after schedule paused and no more + /* Exit main loop after (schedule paused and) no more * packets received */ break; } @@ -1079,7 +1221,9 @@ static int rx_thread(void *arg) } if (num == 0) { - rx_timeouts++; + if (direct_rx == 0) + rx_timeouts++; + continue; } @@ -1090,8 +1234,10 @@ static int rx_thread(void *arg) bytes = 0; for (i = 0; i < num; i++) { - pkt = odp_packet_from_event(ev[i]); - bytes += odp_packet_len(pkt); + bytes += odp_packet_len(pkt[i]); + + if (ts_off) + get_timestamp(pkt[i], ts_off, &rx_lat_data, rx_ts); } rx_packets += num; @@ -1099,7 +1245,7 @@ static int rx_thread(void *arg) if (odp_unlikely(periodic_stat)) { /* All packets from the same queue are from the same pktio interface */ - int index = odp_packet_input_index(odp_packet_from_event(ev[0])); + int index = odp_packet_input_index(pkt[0]); if (index >= 0) { int if_idx = global->if_from_pktio_idx[index]; @@ -1108,22 +1254,26 @@ static int rx_thread(void *arg) } } - odp_event_free_multi(ev, num); + odp_packet_free_multi(pkt, num); } if (clock_started) nsec = odp_time_diff_ns(t2, t1); /* Update stats*/ - global->stat[thr].time_nsec = nsec; - global->stat[thr].rx_timeouts = rx_timeouts; - global->stat[thr].rx_packets = rx_packets; - global->stat[thr].rx_bytes = rx_bytes; + global->stat[thr].time_nsec = nsec; + global->stat[thr].rx_timeouts = rx_timeouts; + global->stat[thr].rx_packets = rx_packets; + global->stat[thr].rx_bytes = rx_bytes; + global->stat[thr].rx_lat_nsec = rx_lat_data.nsec; + global->stat[thr].rx_lat_min_nsec = rx_lat_data.min; + global->stat[thr].rx_lat_max_nsec = rx_lat_data.max; + global->stat[thr].rx_lat_packets = rx_lat_data.packets; return ret; } -static void drain_queues(test_global_t *global) +static void drain_scheduler(test_global_t *global) { odp_event_t ev; uint64_t wait_time = odp_schedule_wait_time(100 * ODP_TIME_MSEC_IN_NS); @@ -1134,6 +1284,26 @@ static void drain_queues(test_global_t *global) } } +static void drain_direct_input(test_global_t *global) +{ + odp_pktin_queue_t pktin; + odp_packet_t pkt; + int i, j; + int num_pktio = global->test_options.num_pktio; + int num_rx = global->test_options.num_rx; + + for (i = 0; i < num_pktio; i++) { + for (j = 0; j < num_rx; j++) { + pktin = global->pktio[i].pktin[j]; + + while (odp_pktin_recv(pktin, &pkt, 1) == 1) { + global->drained++; + odp_packet_free(pkt); + } + } + } +} + static int init_packets(test_global_t *global, int pktio, odp_packet_t packet[], uint32_t num, uint16_t seq) { @@ -1165,7 +1335,7 @@ static int init_packets(test_global_t *global, int pktio, payload_len = pkt_len - hdr_len; if (seg_len < hdr_len) { - printf("Error: First segment too short %u\n", seg_len); + ODPH_ERR("Error: First segment too short %u\n", seg_len); return -1; } @@ -1217,11 +1387,14 @@ static int init_packets(test_global_t *global, int pktio, udp->length = odp_cpu_to_be_16(payload_len + ODPH_UDPHDR_LEN); udp->chksum = 0; - /* Init UDP payload until the end of the first segment */ u8 = data; u8 += hdr_len; - for (j = 0; j < seg_len - hdr_len; j++) - u8[j] = j; + + if (test_options->fill_pl) { + /* Init UDP payload until the end of the first segment */ + for (j = 0; j < seg_len - hdr_len; j++) + u8[j] = j; + } /* Insert UDP checksum */ odp_packet_l3_offset_set(pkt, l2_len); @@ -1229,7 +1402,9 @@ static int init_packets(test_global_t *global, int pktio, odp_packet_has_eth_set(pkt, 1); odp_packet_has_ipv4_set(pkt, 1); odp_packet_has_udp_set(pkt, 1); - udp->chksum = odph_ipv4_udp_chksum(pkt); + + udp->chksum = !test_options->calc_latency && test_options->calc_cs ? + odph_ipv4_udp_chksum(pkt) : 0; /* Increment port numbers */ if (test_options->c_mode.udp_src) { @@ -1264,12 +1439,12 @@ static inline int update_rand_data(uint8_t *data, uint32_t data_len) int32_t ret = odp_random_data(data, data_len - generated, ODP_RANDOM_BASIC); if (odp_unlikely(ret < 0)) { - printf("Error: odp_random_data() failed: %" PRId32 "\n", ret); + ODPH_ERR("Error: odp_random_data() failed: %" PRId32 "\n", ret); return -1; } else if (odp_unlikely(ret == 0)) { retries++; if (odp_unlikely(retries > MAX_RAND_RETRIES)) { - printf("Error: Failed to create random data\n"); + ODPH_ERR("Error: Failed to create random data\n"); return -1; } continue; @@ -1280,13 +1455,23 @@ static inline int update_rand_data(uint8_t *data, uint32_t data_len) return 0; } +static inline void set_timestamp(odp_packet_t pkt, uint32_t ts_off, odp_bool_t calc_cs) +{ + const ts_data_t ts_data = { .magic = TS_MAGIC, .tx_ts = odp_time_global_ns() }; + odph_udphdr_t *udp = odp_packet_l4_ptr(pkt, NULL); + + (void)odp_packet_copy_from_mem(pkt, ts_off, sizeof(ts_data), &ts_data); + udp->chksum = calc_cs ? odph_ipv4_udp_chksum(pkt) : 0; +} + static inline int send_burst(odp_pktout_queue_t pktout, odp_packet_t pkt[], - int burst_size, odp_bool_t use_rand_len, uint32_t pkts_per_pktio, + int burst_size, odp_bool_t use_rand_len, odp_bool_t use_refs, + odp_bool_t calc_cs, uint32_t ts_off, uint32_t pkts_per_pktio, uint64_t *sent_bytes) { - int i, sent; + int i; int ret = 0; int num = burst_size; - odp_packet_t pkt_ref[burst_size]; + odp_packet_t out_pkt[burst_size]; static __thread int rand_idx = RAND_16BIT_WORDS; static __thread uint16_t rand_data[RAND_16BIT_WORDS]; uint64_t bytes_total = 0; @@ -1307,34 +1492,49 @@ static inline int send_burst(odp_pktout_queue_t pktout, odp_packet_t pkt[], idx = rand_data[rand_idx++] % pkts_per_pktio; } - pkt_ref[i] = odp_packet_ref_static(pkt[idx]); - if (odp_unlikely(pkt_ref[i] == ODP_PACKET_INVALID)) { - num = i; - break; + if (use_refs) { + out_pkt[i] = odp_packet_ref_static(pkt[idx]); + if (odp_unlikely(out_pkt[i] == ODP_PACKET_INVALID)) { + num = i; + break; + } + } else { + out_pkt[i] = pkt[idx]; + pkt[idx] = ODP_PACKET_INVALID; + + if (ts_off) + set_timestamp(out_pkt[i], ts_off, calc_cs); } - bytes_total += odp_packet_len(pkt_ref[i]); + bytes_total += odp_packet_len(out_pkt[i]); } - if (odp_unlikely(num == 0)) { - *sent_bytes = 0; - return ret; + if (odp_unlikely(num < burst_size)) { + if (!use_refs) + for (int i = 0; i < burst_size; i++) + if (pkt[i] != ODP_PACKET_INVALID) + odp_packet_free(pkt[i]); + + if (num == 0) { + *sent_bytes = 0; + return ret; + } } - sent = odp_pktout_send(pktout, pkt_ref, num); + ret = odp_pktout_send(pktout, out_pkt, num); - if (odp_unlikely(sent < 0)) - sent = 0; + if (odp_unlikely(ret < 0)) + ret = 0; - if (odp_unlikely(sent != num)) { - uint32_t num_drop = num - sent; + if (odp_unlikely(ret != num)) { + uint32_t num_drop = num - ret; - for (i = sent; i < num; i++) - bytes_total -= odp_packet_len(pkt_ref[i]); - odp_packet_free_multi(&pkt_ref[sent], num_drop); + for (i = ret; i < num; i++) + bytes_total -= odp_packet_len(out_pkt[i]); + odp_packet_free_multi(&out_pkt[ret], num_drop); } *sent_bytes = bytes_total; - return sent; + return ret; } static int alloc_test_packets(odp_pool_t pool, odp_packet_t *pkt_tbl, int num_pkt, @@ -1370,7 +1570,7 @@ static int alloc_test_packets(odp_pool_t pool, odp_packet_t *pkt_tbl, int num_pk pkt_tbl[num_alloc] = odp_packet_alloc(pool, pkt_len); if (pkt_tbl[num_alloc] == ODP_PACKET_INVALID) { - printf("Error: Alloc of %dB packet failed\n", pkt_len); + ODPH_ERR("Error: Alloc of %dB packet failed\n", pkt_len); break; } num_alloc++; @@ -1386,14 +1586,41 @@ static int alloc_test_packets(odp_pool_t pool, odp_packet_t *pkt_tbl, int num_pk } num_alloc = odp_packet_alloc_multi(pool, test_options->pkt_len, pkt_tbl, num_pkt); if (num_alloc != num_pkt) - printf("Error: Alloc of %u packets failed\n", num_pkt); + ODPH_ERR("Error: Alloc of %u packets failed\n", num_pkt); return num_alloc; } +static int allocate_and_init_packets(odp_pool_t pool, odp_packet_t *pkt_tbl, int num_pkt, + int pkts_per_pktio, test_options_t *test_options, + test_global_t *global) +{ + int num_alloc, num_pktio = test_options->num_pktio; + + num_alloc = alloc_test_packets(pool, pkt_tbl, num_pkt, pkts_per_pktio, test_options); + + if (num_alloc != num_pkt) + goto err; + + for (int i = 0; i < num_pktio; i++) { + int f = i * pkts_per_pktio; + + if (init_packets(global, i, &pkt_tbl[f], pkts_per_pktio, f)) + goto err; + } + + return 0; + +err: + if (num_alloc > 0) + odp_packet_free_multi(pkt_tbl, num_alloc); + + return -1; +} + static int tx_thread(void *arg) { - int i, thr, tx_thr, num_alloc; + int i, thr, tx_thr; uint32_t exit_test; odp_time_t t1, t2, next_tmo; uint64_t diff_ns, t1_nsec; @@ -1413,10 +1640,14 @@ static int tx_thread(void *arg) int bursts = test_options->bursts; uint32_t num_tx = test_options->num_tx; odp_bool_t use_rand_len = test_options->use_rand_pkt_len; + odp_bool_t use_refs = test_options->use_refs; + odp_bool_t is_allocd = false; + odp_bool_t calc_cs = test_options->calc_cs; int num_pktio = test_options->num_pktio; int num_pkt; odp_pktout_queue_t pktout[num_pktio]; uint32_t pkts_per_pktio = bursts * burst_size; + uint32_t ts_off = test_options->calc_latency ? test_options->hdr_len : 0; if (use_rand_len) { uint32_t pkt_sizes = test_options->rand_pkt_len_max - @@ -1435,22 +1666,14 @@ static int tx_thread(void *arg) tx_thr = thread_arg->tx_thr; global->stat[thr].thread_type = TX_THREAD; - /* Preallocate test packets */ - num_alloc = alloc_test_packets(pool, pkt, num_pkt, pkts_per_pktio, test_options); - if (num_alloc != num_pkt) - ret = -1; - - /* Initialize packets per pktio interface */ - for (i = 0; ret == 0 && i < num_pktio; i++) { - int f = i * pkts_per_pktio; - + for (i = 0; i < num_pktio; i++) pktout[i] = thread_arg->pktout[i]; - if (init_packets(global, i, &pkt[f], pkts_per_pktio, f)) { - ret = -1; - break; - } - } + if (use_refs) { + ret = allocate_and_init_packets(pool, pkt, num_pkt, pkts_per_pktio, test_options, + global); + is_allocd = !ret; + } /* Start all workers at the same time */ odp_barrier_wait(&global->barrier); @@ -1477,6 +1700,14 @@ static int tx_thread(void *arg) } tx_timeouts++; + if (!use_refs) { + if (odp_unlikely(allocate_and_init_packets(pool, pkt, num_pkt, + pkts_per_pktio, test_options, + global) < 0)) { + ret = -1; + break; + } + } /* Send bursts to each pktio */ for (i = 0; i < num_pktio; i++) { int sent, j; @@ -1484,10 +1715,9 @@ static int tx_thread(void *arg) uint64_t sent_bytes; for (j = 0; j < bursts; j++) { - sent = send_burst(pktout[i], - &pkt[first + j * burst_size], - burst_size, use_rand_len, - pkts_per_pktio, &sent_bytes); + sent = send_burst(pktout[i], &pkt[first + j * burst_size], + burst_size, use_rand_len, use_refs, calc_cs, + ts_off, pkts_per_pktio, &sent_bytes); if (odp_unlikely(sent < 0)) { ret = -1; @@ -1510,8 +1740,8 @@ static int tx_thread(void *arg) t2 = odp_time_local(); diff_ns = odp_time_diff_ns(t2, t1); - if (num_alloc > 0) - odp_packet_free_multi(pkt, num_alloc); + if (is_allocd) + odp_packet_free_multi(pkt, num_pkt); /* Update stats */ global->stat[thr].time_nsec = diff_ns; @@ -1541,6 +1771,10 @@ static int start_workers(test_global_t *global, odp_instance_t instance) /* Receive threads */ for (i = 0; i < num_rx; i++) { + /* In direct mode, dedicate a pktin queue per pktio interface (per RX thread) */ + for (j = 0; test_options->direct_rx && j < num_pktio; j++) + global->thread_arg[i].pktin[j] = global->pktio[j].pktin[i]; + odph_thread_param_init(&thr_param[i]); thr_param[i].start = rx_thread; thr_param[i].arg = &global->thread_arg[i]; @@ -1572,7 +1806,7 @@ static int start_workers(test_global_t *global, odp_instance_t instance) num_cpu); if (ret != num_cpu) { - printf("Error: thread create failed %i\n", ret); + ODPH_ERR("Error: thread create failed %i\n", ret); return -1; } @@ -1637,6 +1871,28 @@ static void periodic_print_loop(test_global_t *global) } } +static void print_humanised_time(double time_nsec) +{ + if (time_nsec > ODP_TIME_SEC_IN_NS) + printf("%.2f s\n", time_nsec / ODP_TIME_SEC_IN_NS); + else if (time_nsec > ODP_TIME_MSEC_IN_NS) + printf("%.2f ms\n", time_nsec / ODP_TIME_MSEC_IN_NS); + else if (time_nsec > ODP_TIME_USEC_IN_NS) + printf("%.2f us\n", time_nsec / ODP_TIME_USEC_IN_NS); + else + printf("%.0f ns\n", time_nsec); +} + +static void print_humanised_latency(double lat_nsec, double lat_min_nsec, double lat_max_nsec) +{ + printf(" rx ave packet latency: "); + print_humanised_time(lat_nsec); + printf(" rx min packet latency: "); + print_humanised_time(lat_min_nsec); + printf(" rx max packet latency: "); + print_humanised_time(lat_max_nsec); +} + static int print_final_stat(test_global_t *global) { int i, num_thr; @@ -1648,6 +1904,10 @@ static int print_final_stat(test_global_t *global) uint64_t rx_pkt_sum = 0; uint64_t rx_byte_sum = 0; uint64_t rx_tmo_sum = 0; + uint64_t rx_lat_nsec_sum = 0; + uint64_t rx_lat_min_nsec = UINT64_MAX; + uint64_t rx_lat_max_nsec = 0; + uint64_t rx_lat_pkt_sum = 0; uint64_t tx_nsec_sum = 0; uint64_t tx_pkt_sum = 0; uint64_t tx_byte_sum = 0; @@ -1657,6 +1917,7 @@ static int print_final_stat(test_global_t *global) double rx_byte_per_sec = 0.0; double rx_pkt_len = 0.0; double rx_sec = 0.0; + double rx_ave_lat_nsec = 0.0; double tx_pkt_per_sec = 0.0; double tx_byte_per_sec = 0.0; double tx_sec = 0.0; @@ -1702,15 +1963,22 @@ static int print_final_stat(test_global_t *global) for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) { if (global->stat[i].thread_type == RX_THREAD) { - rx_tmo_sum += global->stat[i].rx_timeouts; - rx_pkt_sum += global->stat[i].rx_packets; - rx_byte_sum += global->stat[i].rx_bytes; - rx_nsec_sum += global->stat[i].time_nsec; - + rx_tmo_sum += global->stat[i].rx_timeouts; + rx_pkt_sum += global->stat[i].rx_packets; + rx_byte_sum += global->stat[i].rx_bytes; + rx_nsec_sum += global->stat[i].time_nsec; + rx_lat_nsec_sum += global->stat[i].rx_lat_nsec; + rx_lat_pkt_sum += global->stat[i].rx_lat_packets; + + if (global->stat[i].rx_lat_min_nsec < rx_lat_min_nsec) + rx_lat_min_nsec = global->stat[i].rx_lat_min_nsec; + + if (global->stat[i].rx_lat_max_nsec > rx_lat_max_nsec) + rx_lat_max_nsec = global->stat[i].rx_lat_max_nsec; } else if (global->stat[i].thread_type == TX_THREAD) { tx_tmo_sum += global->stat[i].tx_timeouts; tx_pkt_sum += global->stat[i].tx_packets; - tx_byte_sum += global->stat[i].tx_bytes; + tx_byte_sum += global->stat[i].tx_bytes; tx_drop_sum += global->stat[i].tx_drops; tx_nsec_sum += global->stat[i].time_nsec; } @@ -1746,6 +2014,9 @@ static int print_final_stat(test_global_t *global) if (rx_pkt_sum) rx_pkt_len = (double)rx_byte_sum / rx_pkt_sum; + if (rx_lat_pkt_sum) + rx_ave_lat_nsec = (double)rx_lat_nsec_sum / rx_lat_pkt_sum; + printf("TOTAL (%i rx and %i tx threads)\n", num_rx, num_tx); printf(" rx timeouts: %" PRIu64 "\n", rx_tmo_sum); printf(" rx time spent (sec): %.3f\n", rx_sec); @@ -1755,6 +2026,10 @@ static int print_final_stat(test_global_t *global) printf(" rx packets per thr per sec: %.1f\n", rx_pkt_per_sec); printf(" rx packets per sec: %.1f\n", num_rx * rx_pkt_per_sec); printf(" rx ave packet len: %.1f\n", rx_pkt_len); + + if (rx_lat_pkt_sum) + print_humanised_latency(rx_ave_lat_nsec, rx_lat_min_nsec, rx_lat_max_nsec); + printf(" rx Mbit/s: %.1f\n", rx_mbit_per_sec); printf("\n"); printf(" tx timeouts: %" PRIu64 "\n", tx_tmo_sum); @@ -1814,13 +2089,13 @@ int main(int argc, char **argv) /* Init ODP before calling anything else */ if (odp_init_global(&instance, &init, NULL)) { - printf("Error: Global init failed.\n"); + ODPH_ERR("Error: Global init failed.\n"); return 1; } /* Init this thread */ if (odp_init_local(instance, ODP_THREAD_CONTROL)) { - printf("Error: Local init failed.\n"); + ODPH_ERR("Error: Local init failed.\n"); return 1; } @@ -1828,7 +2103,7 @@ int main(int argc, char **argv) ODP_CACHE_LINE_SIZE, 0); if (shm == ODP_SHM_INVALID) { - printf("Error: SHM reserve failed.\n"); + ODPH_ERR("Error: SHM reserve failed.\n"); return 1; } @@ -1848,7 +2123,9 @@ int main(int argc, char **argv) odp_sys_info_print(); - odp_schedule_config(NULL); + /* Avoid all scheduler API calls in direct input mode */ + if (global->test_options.direct_rx == 0) + odp_schedule_config(NULL); if (set_num_cpu(global)) { ret = 1; @@ -1882,7 +2159,10 @@ int main(int argc, char **argv) if (stop_pktios(global)) ret = 1; - drain_queues(global); + if (global->test_options.direct_rx) + drain_direct_input(global); + else + drain_scheduler(global); if (close_pktios(global)) ret = 1; @@ -1892,17 +2172,17 @@ int main(int argc, char **argv) term: if (odp_shm_free(shm)) { - printf("Error: SHM free failed.\n"); + ODPH_ERR("Error: SHM free failed.\n"); return 1; } if (odp_term_local()) { - printf("Error: term local failed.\n"); + ODPH_ERR("Error: term local failed.\n"); return 1; } if (odp_term_global(instance)) { - printf("Error: term global failed.\n"); + ODPH_ERR("Error: term global failed.\n"); return 1; } diff --git a/test/validation/api/Makefile.am b/test/validation/api/Makefile.am index 91e600577..c08ff3d6e 100644 --- a/test/validation/api/Makefile.am +++ b/test/validation/api/Makefile.am @@ -88,9 +88,4 @@ $(TESTENV): echo "$(TESTS_ENVIRONMENT)" >> $@ echo "$(LOG_COMPILER)" >> $@ -if test_installdir -installcheck-local: - $(DESTDIR)/$(testdir)/run-test.sh $(TESTNAME) -endif - .NOTPARALLEL: diff --git a/test/validation/api/classification/odp_classification_test_pmr.c b/test/validation/api/classification/odp_classification_test_pmr.c index 9a2e32f4b..280e525af 100644 --- a/test/validation/api/classification/odp_classification_test_pmr.c +++ b/test/validation/api/classification/odp_classification_test_pmr.c @@ -836,8 +836,18 @@ static void classification_test_pmr_term_vlan_id_x(void) pmr_param.match.mask = &mask; pmr_param.val_sz = sizeof(val); + /* Single VLAN */ pkt_info = default_pkt_info; pkt_info.vlan = true; + pkt = create_packet(pkt_info); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL); + vlan_x = (odph_vlanhdr_t *)(eth + 1); + vlan_x->tci = val; + + test_pmr(&pmr_param, pkt, MATCH); + + /* Two VLANs */ pkt_info.vlan_qinq = true; pkt = create_packet(pkt_info); CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); @@ -848,6 +858,7 @@ static void classification_test_pmr_term_vlan_id_x(void) test_pmr(&pmr_param, pkt, MATCH); + /* No VLAN */ pkt = create_packet(default_pkt_info); CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); @@ -941,8 +952,18 @@ static void classification_test_pmr_term_eth_type_x(void) pmr_param.match.mask = &mask; pmr_param.val_sz = sizeof(val); + /* Single VLAN */ pkt_info = default_pkt_info; pkt_info.vlan = true; + pkt = create_packet(pkt_info); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL); + vlan_x = (odph_vlanhdr_t *)(eth + 1); + vlan_x->tci = odp_cpu_to_be_16(0x123); + + test_pmr(&pmr_param, pkt, MATCH); + + /* Two VLANs */ pkt_info.vlan_qinq = true; pkt = create_packet(pkt_info); CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); @@ -950,10 +971,10 @@ static void classification_test_pmr_term_eth_type_x(void) vlan_x = (odph_vlanhdr_t *)(eth + 1); vlan_x++; vlan_x->tci = odp_cpu_to_be_16(0x123); - vlan_x->type = val; test_pmr(&pmr_param, pkt, MATCH); + /* No VLAN */ pkt = create_packet(default_pkt_info); CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); diff --git a/test/validation/api/packet/packet.c b/test/validation/api/packet/packet.c index d81fba3de..1ae973d79 100644 --- a/test/validation/api/packet/packet.c +++ b/test/validation/api/packet/packet.c @@ -28,6 +28,8 @@ ODP_STATIC_ASSERT(PACKET_POOL_NUM_SEG > 1 && /* Number of packets in parse test */ #define PARSE_TEST_NUM_PKT 10 +/* Maximum offset to Ethernet in parse tests */ +#define MAX_PARSE_L2_OFFSET 207 /* Default packet vector size */ #define PKT_VEC_SIZE 64 @@ -66,7 +68,7 @@ static struct udata_struct { static struct { odp_pool_t pool; odp_proto_chksums_t all_chksums; - uint32_t offset_zero[PARSE_TEST_NUM_PKT]; + uint32_t l2_offset[PARSE_TEST_NUM_PKT]; } parse_test; static uint32_t parse_test_pkt_len[] = { @@ -3451,6 +3453,7 @@ static int packet_parse_suite_init(void) if (max_len < parse_test_pkt_len[i]) max_len = parse_test_pkt_len[i]; } + max_len += MAX_PARSE_L2_OFFSET; odp_pool_param_init(¶m); @@ -3470,9 +3473,6 @@ static int packet_parse_suite_init(void) parse_test.all_chksums.chksum.tcp = 1; parse_test.all_chksums.chksum.sctp = 1; - for (i = 0; i < PARSE_TEST_NUM_PKT; i++) - parse_test.offset_zero[i] = 0; - return 0; } @@ -3488,12 +3488,24 @@ static void parse_test_alloc(odp_packet_t pkt[], const uint8_t test_packet[], uint32_t len, int num_pkt) { int ret, i; + static uint32_t l2_offset[PARSE_TEST_NUM_PKT] = {0 /* must be zero */, + 2, 8, 12, 19, 36, 64, 120, MAX_PARSE_L2_OFFSET}; - ret = odp_packet_alloc_multi(parse_test.pool, len, pkt, num_pkt); - CU_ASSERT_FATAL(ret == num_pkt); + CU_ASSERT_FATAL(num_pkt <= PARSE_TEST_NUM_PKT); for (i = 0; i < num_pkt; i++) { - ret = odp_packet_copy_from_mem(pkt[i], 0, len, test_packet); + uint32_t offs = l2_offset[i]; + uint32_t data = 0; + + parse_test.l2_offset[i] = offs; + pkt[i] = odp_packet_alloc(parse_test.pool, len + offs); + CU_ASSERT_FATAL(pkt[i] != ODP_PACKET_INVALID); + + if (offs > 0) { + ret = fill_data_forward(pkt[i], 0, offs, &data); + CU_ASSERT(ret == 0); + } + ret = odp_packet_copy_from_mem(pkt[i], offs, len, test_packet); CU_ASSERT_FATAL(ret == 0); } } @@ -3522,7 +3534,7 @@ static void parse_eth_ipv4_udp(void) parse.chksums = parse_test.all_chksums; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -3566,7 +3578,7 @@ static void parse_eth_snap_ipv4_udp(void) parse.chksums = parse_test.all_chksums; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -3599,7 +3611,7 @@ static void parse_ipv4_udp(void) sizeof(test_packet_ipv4_udp), num_pkt); for (i = 0; i < num_pkt; i++) - offset[i] = 14; + offset[i] = parse_test.l2_offset[i] + 14; parse.proto = ODP_PROTO_IPV4; parse.last_layer = ODP_PROTO_LAYER_L4; @@ -3639,7 +3651,7 @@ static void parse_eth_ipv4_tcp(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -3675,7 +3687,7 @@ static void parse_eth_ipv6_udp(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -3705,7 +3717,7 @@ static void parse_eth_ipv6_tcp(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -3735,7 +3747,7 @@ static void parse_eth_vlan_ipv4_udp(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -3766,7 +3778,7 @@ static void parse_eth_vlan_ipv6_udp(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -3803,7 +3815,7 @@ static void parse_eth_vlan_qinq_ipv4_udp(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -3835,7 +3847,7 @@ static void parse_eth_arp(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -3867,7 +3879,7 @@ static void parse_eth_ipv4_icmp(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -3898,7 +3910,7 @@ static void parse_eth_ipv6_icmp(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -3929,7 +3941,7 @@ static void parse_eth_ipv4_sctp(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -3960,7 +3972,7 @@ static void parse_eth_ipv4_ipsec_ah(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -3991,7 +4003,7 @@ static void parse_eth_ipv4_ipsec_esp(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -4022,7 +4034,7 @@ static void parse_eth_ipv6_ipsec_ah(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -4059,7 +4071,7 @@ static void parse_eth_ipv6_ipsec_esp(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -4090,7 +4102,7 @@ static void parse_mcast_eth_ipv4_udp(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -4124,7 +4136,7 @@ static void parse_bcast_eth_ipv4_udp(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -4159,7 +4171,7 @@ static void parse_mcast_eth_ipv6_udp(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -4193,7 +4205,7 @@ static void parse_eth_ipv4_udp_first_frag(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -4225,7 +4237,7 @@ static void parse_eth_ipv4_udp_last_frag(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -4257,7 +4269,7 @@ static void parse_eth_ipv4_rr_nop_icmp(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { @@ -4292,7 +4304,7 @@ static void parse_result(void) parse.chksums.all_chksum = 0; CU_ASSERT(odp_packet_parse(pkt[0], 0, &parse) == 0); - CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.offset_zero, + CU_ASSERT(odp_packet_parse_multi(&pkt[1], parse_test.l2_offset + 1, num_pkt - 1, &parse) == (num_pkt - 1)); for (i = 0; i < num_pkt; i++) { diff --git a/test/validation/api/traffic_mngr/traffic_mngr.c b/test/validation/api/traffic_mngr/traffic_mngr.c index 0679e215c..038a4da8b 100644 --- a/test/validation/api/traffic_mngr/traffic_mngr.c +++ b/test/validation/api/traffic_mngr/traffic_mngr.c @@ -4688,6 +4688,8 @@ static void traffic_mngr_test_queue_stats(void) num_rcv_pkts = receive_pkts(odp_tm_systems[0], rcv_pktin, pkts_sent, 1 * GBPS); + odp_tm_stats_print(odp_tm_systems[0]); + CU_ASSERT(odp_tm_queue_stats(tm_queue, &stats_stop) == 0); if (capa.queue_stats.counter.packets) |