diff options
author | Matias Elo <matias.elo@nokia.com> | 2021-10-11 16:37:51 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-10-11 16:37:51 +0300 |
commit | 50c7b605b97474d26fcf600b4061968e3543b45b (patch) | |
tree | bb5541aab05c99d762a054a9f1fe64cd002d7f86 | |
parent | 54ca03915ddf070a3c67f6a3c89f442c202a1adb (diff) | |
parent | ec77bf59d3407278625c28a8d47c3dab9627a91f (diff) |
Merge ODP v1.32.0.0v1.32.0.0_DPDK_19.11
Merge ODP linux-generic v1.32.0.0 into ODP-DPDK.
143 files changed, 3069 insertions, 1747 deletions
diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 28150c4de..5992c507e 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -7,7 +7,7 @@ on: jobs: Documentation: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - name: Install dependencies @@ -44,7 +44,7 @@ jobs: - name: Deploy env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: crazy-max/ghaction-github-pages@v2.2.0 + uses: crazy-max/ghaction-github-pages@v2 with: allow_empty_commit: false build_dir: ./doc/gh-pages @@ -1,3 +1,54 @@ +== OpenDataPlane (1.32.0.0) +=== Backward incompatible API changes +==== IPsec +* Added destination queue capabilities `odp_ipsec_capability_t.queue_type_sched` +and `odp_ipsec_capability_t.queue_type_plain` +* Modified specification to not promise the original packet back after all error +cases + +=== Backward compatible API changes +==== IPsec +* Added original ESP/AH packet length into IPsec packet result +(`odp_ipsec_packet_result_t.orig_ip_len`) + +==== Packet +* Added `odp_packet_reass_info()` function for reading completed reassembly +details + +==== Std +* Moved all contents of support and feature modules into ODP Std module + +=== Helper (1.2.0) +==== Backward incompatible changes +===== CLI +* Replaced `odph_cli_start()` function with `odph_cli_run()`, which doesn't +automatically create a separate thread for running the CLI server. + +===== Linux +* Deprecated `odph_odpthread_t` and `odph_odpthread_params_t` types +* Added `odph_thread_param_init()` function for initializing thread parameters +* Added `odph_thread_common_param_init()` function for initializing thread +common parameters + +==== Backward compatible changes +===== CLI +* Added `odph_cli_param_t.hostname` parameter for providing a custom CLI prompt +hostname +* Added `odph_cli_log_va()` function for user defined CLI commands + +===== Linux +* Added `odph_thread_param_t.stack_size` parameter for configuring minimum +thread stack size +* Added `odph_thread_common_param_t.sync_timeout` parameter for configuring +synchronized thread creation timeout + +=== Implementation +==== Crypto +* AES-EEA2 and AES-EIA2 crypto algorithms that operate in bit mode interpret the +crypto/auth offset now as bits as specified in the ODP API. This correction +requires corresponding changes in applications that have relied on the old, +incorrect behavior that interpreted the offset in bytes. + == OpenDataPlane (1.31.0.0) === Backward incompatible API changes ==== Traffic Manager diff --git a/DEPENDENCIES b/DEPENDENCIES index 5b5f66904..b0762067e 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -403,3 +403,14 @@ Prerequisites for building the OpenDataPlane (ODP) API To have this tool also check spelling you need codespell. # Debian/Ubuntu $ sudo apt-get install codespell + +8.0 Command Line Interface (optional) + + ODP applications (see e.g. ./example/cli) may use CLI helper (./helper/include/odp/helper/cli.h) + to provide a command line interface to the user. CLI helper depends on libcli library. + + # Debian/Ubuntu + $ sudo apt-get install libcli-dev + + # CentOS/RedHat/Fedora + $ sudo yum install libcli-devel diff --git a/configure.ac b/configure.ac index d8f9f487b..912bc61ef 100644 --- a/configure.ac +++ b/configure.ac @@ -3,7 +3,7 @@ AC_PREREQ([2.5]) # ODP API version ########################################################################## m4_define([odpapi_generation_version], [1]) -m4_define([odpapi_major_version], [31]) +m4_define([odpapi_major_version], [32]) m4_define([odpapi_minor_version], [0]) m4_define([odpapi_point_version], [1]) m4_define([odpapi_version], @@ -21,8 +21,8 @@ AC_SUBST(ODP_VERSION_API_MINOR) # Helper library version ########################################################################## m4_define([odph_version_generation], [1]) -m4_define([odph_version_major], [1]) -m4_define([odph_version_minor], [1]) +m4_define([odph_version_major], [2]) +m4_define([odph_version_minor], [0]) m4_define([odph_version], [odph_version_generation.odph_version_major.odph_version_minor]) diff --git a/doc/implementers-guide/implementers-guide.adoc b/doc/implementers-guide/implementers-guide.adoc index 922188770..398934482 100644 --- a/doc/implementers-guide/implementers-guide.adoc +++ b/doc/implementers-guide/implementers-guide.adoc @@ -556,9 +556,9 @@ could (hopefully) be as simple as changing the OS related helper lib. In the linux helper, two functions are given to create and join ODP threads: -`odph_odpthreads_create()` +`odph_thread_create()` -`odph_odpthreads_join()` +`odph_thread_join()` These two functions abstract what an ODP thread really is and their usage is recommended as they would be implemented in other OS`s helper lib. diff --git a/example/classifier/odp_classifier.c b/example/classifier/odp_classifier.c index ebac378a6..2c36fce0e 100644 --- a/example/classifier/odp_classifier.c +++ b/example/classifier/odp_classifier.c @@ -683,8 +683,8 @@ int main(int argc, char *argv[]) /* Create and init worker threads */ memset(thread_tbl, 0, sizeof(thread_tbl)); - memset(&thr_common, 0, sizeof(thr_common)); - memset(&thr_param, 0, sizeof(thr_param)); + odph_thread_common_param_init(&thr_common); + odph_thread_param_init(&thr_param); thr_param.start = pktio_receive_thread; thr_param.arg = args; diff --git a/example/cli/odp_cli.c b/example/cli/odp_cli.c index 597d01812..48badb5cd 100644 --- a/example/cli/odp_cli.c +++ b/example/cli/odp_cli.c @@ -96,6 +96,17 @@ static void my_cmd(int argc, char *argv[]) odph_cli_log("argv[%d]: %s\n", i, argv[i]); } +static int cli_server(void *arg ODP_UNUSED) +{ + /* Run CLI server. */ + if (odph_cli_run()) { + ODPH_ERR("odph_cli_run() failed.\n"); + exit(EXIT_FAILURE); + } + + return 0; +} + int main(int argc, char *argv[]) { signal(SIGINT, sig_handler); @@ -161,9 +172,30 @@ int main(int argc, char *argv[]) exit(EXIT_FAILURE); } - /* Start CLI server. */ - if (odph_cli_start()) { - ODPH_ERR("CLI start failed.\n"); + /* Create server thread. */ + + odp_cpumask_t cpumask; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; + odph_thread_t thr_server; + + if (odp_cpumask_default_control(&cpumask, 1) != 1) { + ODPH_ERR("Failed to get default CPU mask.\n"); + exit(EXIT_FAILURE); + } + + odph_thread_common_param_init(&thr_common); + thr_common.instance = inst; + thr_common.cpumask = &cpumask; + + odph_thread_param_init(&thr_param); + thr_param.thr_type = ODP_THREAD_CONTROL; + thr_param.start = cli_server; + + memset(&thr_server, 0, sizeof(thr_server)); + + if (odph_thread_create(&thr_server, &thr_common, &thr_param, 1) != 1) { + ODPH_ERR("Failed to create server thread.\n"); exit(EXIT_FAILURE); } @@ -182,6 +214,12 @@ int main(int argc, char *argv[]) exit(EXIT_FAILURE); } + /* Wait for server thread to exit. */ + if (odph_thread_join(&thr_server, 1) != 1) { + ODPH_ERR("Failed to join server thread.\n"); + exit(EXIT_FAILURE); + } + /* Terminate CLI helper. */ if (odph_cli_term()) { ODPH_ERR("CLI helper termination failed.\n"); diff --git a/example/debug/odp_debug.c b/example/debug/odp_debug.c index 8b1ef52ba..5c0beef69 100644 --- a/example/debug/odp_debug.c +++ b/example/debug/odp_debug.c @@ -243,52 +243,65 @@ static int pool_debug(void) static int queue_debug(void) { - odp_queue_t queue; odp_queue_param_t param; const char *name; + int i; + int num = 3; + odp_queue_t queue[num]; - name = "debug_plain_queue"; + name = "plain_queue"; odp_queue_param_init(¶m); param.type = ODP_QUEUE_TYPE_PLAIN; - queue = odp_queue_create(name, ¶m); + queue[0] = odp_queue_create(name, ¶m); - if (queue == ODP_QUEUE_INVALID) { + if (queue[0] == ODP_QUEUE_INVALID) { ODPH_ERR("Queue create failed: %s\n", name); return -1; } printf("\n"); - odp_queue_print_all(); + odp_queue_print(queue[0]); - printf("\n"); - odp_queue_print(queue); + name = "parallel_sched_queue"; + odp_queue_param_init(¶m); + param.type = ODP_QUEUE_TYPE_SCHED; - if (odp_queue_destroy(queue)) { - ODPH_ERR("Queue destroy failed: %s\n", name); + queue[1] = odp_queue_create(name, ¶m); + + if (queue[1] == ODP_QUEUE_INVALID) { + ODPH_ERR("Queue create failed: %s\n", name); return -1; } - name = "debug_sched_queue"; - odp_queue_param_init(¶m); - param.type = ODP_QUEUE_TYPE_SCHED; + printf("\n"); + odp_queue_print(queue[1]); - queue = odp_queue_create(name, ¶m); + name = "atomic_sched_queue"; + param.sched.sync = ODP_SCHED_SYNC_ATOMIC; + param.sched.prio = ODP_SCHED_PRIO_HIGHEST; - if (queue == ODP_QUEUE_INVALID) { + queue[2] = odp_queue_create(name, ¶m); + + if (queue[2] == ODP_QUEUE_INVALID) { ODPH_ERR("Queue create failed: %s\n", name); return -1; } printf("\n"); - odp_queue_print(queue); + odp_queue_print(queue[2]); + + printf("\n"); + odp_queue_print_all(); printf("\n"); odp_schedule_print(); - if (odp_queue_destroy(queue)) { - ODPH_ERR("Queue destroy failed: %s\n", name); - return -1; + for (i = 0; i < num; i++) { + if (odp_queue_destroy(queue[i])) { + ODPH_ERR("Queue destroy failed: %i\n", i); + return -1; + } } return 0; diff --git a/example/generator/odp_generator.c b/example/generator/odp_generator.c index 469314718..8ad73601f 100644 --- a/example/generator/odp_generator.c +++ b/example/generator/odp_generator.c @@ -1255,10 +1255,10 @@ int main(int argc, char *argv[]) memset(thread_tbl, 0, sizeof(thread_tbl)); /* Init threads params */ - memset(&thr_param, 0, sizeof(thr_param)); + odph_thread_param_init(&thr_param); thr_param.thr_type = ODP_THREAD_WORKER; - memset(&thr_common, 0, sizeof(thr_common)); + odph_thread_common_param_init(&thr_common); thr_common.instance = instance; /* num workers + print thread */ diff --git a/example/ipfragreass/odp_ipfragreass.c b/example/ipfragreass/odp_ipfragreass.c index 00e7b8643..828f11002 100644 --- a/example/ipfragreass/odp_ipfragreass.c +++ b/example/ipfragreass/odp_ipfragreass.c @@ -230,8 +230,9 @@ int main(void) odp_pool_t fragment_pool; odp_shm_t shm; odp_cpumask_t cpumask; - odph_odpthread_t threads[MAX_WORKERS]; - odph_odpthread_params_t thread_params; + odph_thread_t thread_tbl[MAX_WORKERS]; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; odp_packet_t dequeued_pkts[NUM_PACKETS]; odp_event_t ev; odp_u16be_t ip_id = 0; @@ -242,7 +243,6 @@ int main(void) int num_workers = MAX_WORKERS; int reassembled; - memset(&threads, 0, sizeof(threads)); init(&instance, &fragment_pool, &shm, &cpumask, &num_workers); /* Packet generation & fragmentation */ @@ -290,19 +290,25 @@ int main(void) } /* Spawn the worker threads for reassembly */ - memset(&thread_params, 0, sizeof(thread_params)); - thread_params.start = run_worker; - thread_params.arg = 0; - thread_params.thr_type = ODP_THREAD_WORKER; - thread_params.instance = instance; - odph_odpthreads_create(threads, &cpumask, &thread_params); + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &cpumask; + thr_common.share_param = 1; + + odph_thread_param_init(&thr_param); + thr_param.start = run_worker; + thr_param.arg = 0; + thr_param.thr_type = ODP_THREAD_WORKER; + + memset(thread_tbl, 0, sizeof(thread_tbl)); + odph_thread_create(thread_tbl, &thr_common, &thr_param, num_workers); /* Go! */ printf("\n= Starting reassembly...\n"); odp_barrier_wait(&barrier); /* Wait for all threads to complete and output statistics */ - odph_odpthreads_join(threads); + odph_thread_join(thread_tbl, num_workers); for (i = 0; i < num_workers; ++i) printf("=== Thread %02d processed %3d fragments\n", i, thread_stats[i].frags); diff --git a/example/ipsec_api/odp_ipsec.c b/example/ipsec_api/odp_ipsec.c index 5cdb59da3..cebc733ae 100644 --- a/example/ipsec_api/odp_ipsec.c +++ b/example/ipsec_api/odp_ipsec.c @@ -742,7 +742,7 @@ pkt_disposition_e do_ipsec_out_classify(odp_packet_t *ppkt, pkt_ctx_t *ctx) * - Sequence number assignment queue * - Per packet crypto API completion queue * - * @param arg Required by "odph_odpthreads_create", unused + * @param arg Required by "odph_thread_create", unused * * @return NULL (should never return) */ @@ -914,17 +914,19 @@ int main(int argc, char *argv[]) { odph_helper_options_t helper_options; - odph_odpthread_t thread_tbl[MAX_WORKERS]; + odph_thread_t thread_tbl[MAX_WORKERS]; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; int num_workers; int i; int stream_count; odp_shm_t shm; odp_cpumask_t cpumask; char cpumaskstr[ODP_CPUMASK_STR_SIZE]; + odp_ipsec_capability_t ipsec_capa; odp_pool_param_t params; odp_instance_t instance; odp_init_t init_param; - odph_odpthread_params_t thr_params; odp_event_t ev; /* create by default scheduled queues */ @@ -963,6 +965,23 @@ main(int argc, char *argv[]) exit(EXIT_FAILURE); } + if (odp_ipsec_capability(&ipsec_capa)) { + ODPH_ERR("Error: IPsec capability request failed.\n"); + exit(EXIT_FAILURE); + } + + if (queue_create == polled_odp_queue_create) { + if (!ipsec_capa.queue_type_plain) { + ODPH_ERR("Error: Plain type dest queue not supported.\n"); + exit(EXIT_FAILURE); + } + } else { + if (!ipsec_capa.queue_type_sched) { + ODPH_ERR("Error: scheduled type dest queue not supported.\n"); + exit(EXIT_FAILURE); + } + } + /* Reserve memory for args from shared mem */ shm = odp_shm_reserve("shm_args", sizeof(global_data_t), ODP_CACHE_LINE_SIZE, 0); @@ -1060,13 +1079,18 @@ main(int argc, char *argv[]) /* * Create and init worker threads */ + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &cpumask; + thr_common.share_param = 1; + + odph_thread_param_init(&thr_param); + thr_param.start = pktio_thread; + thr_param.arg = NULL; + thr_param.thr_type = ODP_THREAD_WORKER; + memset(thread_tbl, 0, sizeof(thread_tbl)); - memset(&thr_params, 0, sizeof(thr_params)); - thr_params.start = pktio_thread; - thr_params.arg = NULL; - thr_params.thr_type = ODP_THREAD_WORKER; - thr_params.instance = instance; - odph_odpthreads_create(thread_tbl, &cpumask, &thr_params); + odph_thread_create(thread_tbl, &thr_common, &thr_param, num_workers); /* If there are streams attempt to verify them. Otherwise, run until * SIGINT is received. */ @@ -1080,7 +1104,7 @@ main(int argc, char *argv[]) printf("All received\n"); odp_atomic_store_u32(&global->exit_threads, 1); } - odph_odpthreads_join(thread_tbl); + odph_thread_join(thread_tbl, num_workers); /* Stop and close used pktio devices */ for (i = 0; i < global->appl.if_count; i++) { diff --git a/example/ipsec_crypto/odp_ipsec.c b/example/ipsec_crypto/odp_ipsec.c index a55aa6aba..9fec94620 100644 --- a/example/ipsec_crypto/odp_ipsec.c +++ b/example/ipsec_crypto/odp_ipsec.c @@ -1042,7 +1042,7 @@ pkt_disposition_e do_ipsec_out_finish(odp_packet_t pkt, * - Sequence number assignment queue * - Per packet crypto API completion queue * - * @param arg Required by "odph_odpthreads_create", unused + * @param arg Required by "odph_thread_create", unused * * @return NULL (should never return) */ @@ -1203,7 +1203,9 @@ int main(int argc, char *argv[]) { odph_helper_options_t helper_options; - odph_odpthread_t thread_tbl[MAX_WORKERS]; + odph_thread_t thread_tbl[MAX_WORKERS]; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; int num_workers; int i; int stream_count; @@ -1214,7 +1216,6 @@ main(int argc, char *argv[]) odp_pool_param_t params; odp_instance_t instance; odp_init_t init_param; - odph_odpthread_params_t thr_params; /* create by default scheduled queues */ queue_create = odp_queue_create; @@ -1364,13 +1365,18 @@ main(int argc, char *argv[]) /* * Create and init worker threads */ + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &cpumask; + thr_common.share_param = 1; + + odph_thread_param_init(&thr_param); + thr_param.start = pktio_thread; + thr_param.arg = NULL; + thr_param.thr_type = ODP_THREAD_WORKER; + memset(thread_tbl, 0, sizeof(thread_tbl)); - memset(&thr_params, 0, sizeof(thr_params)); - thr_params.start = pktio_thread; - thr_params.arg = NULL; - thr_params.thr_type = ODP_THREAD_WORKER; - thr_params.instance = instance; - odph_odpthreads_create(thread_tbl, &cpumask, &thr_params); + odph_thread_create(thread_tbl, &thr_common, &thr_param, num_workers); /* If there are streams attempt to verify them. Otherwise, run until * SIGINT is received. */ @@ -1384,7 +1390,7 @@ main(int argc, char *argv[]) printf("All received\n"); odp_atomic_store_u32(&global->exit_threads, 1); } - odph_odpthreads_join(thread_tbl); + odph_thread_join(thread_tbl, num_workers); /* Stop and close used pktio devices */ for (i = 0; i < global->appl.if_count; i++) { diff --git a/example/l2fwd_simple/odp_l2fwd_simple.c b/example/l2fwd_simple/odp_l2fwd_simple.c index 94f21ddf2..c9ae94c12 100644 --- a/example/l2fwd_simple/odp_l2fwd_simple.c +++ b/example/l2fwd_simple/odp_l2fwd_simple.c @@ -247,8 +247,8 @@ int main(int argc, char **argv) odp_cpumask_default_worker(&cpumask, MAX_WORKERS); - memset(&thr_common, 0, sizeof(thr_common)); - memset(&thr_param, 0, sizeof(thr_param)); + odph_thread_common_param_init(&thr_common); + odph_thread_param_init(&thr_param); thr_param.start = run_worker; thr_param.thr_type = ODP_THREAD_WORKER; diff --git a/example/l3fwd/odp_l3fwd.c b/example/l3fwd/odp_l3fwd.c index 367e09a24..8debb2010 100644 --- a/example/l3fwd/odp_l3fwd.c +++ b/example/l3fwd/odp_l3fwd.c @@ -82,7 +82,6 @@ typedef struct { typedef struct { app_args_t cmd_args; struct l3fwd_pktio_s l3fwd_pktios[MAX_NB_PKTIO]; - odph_odpthread_t l3fwd_workers[MAX_NB_WORKER]; struct thread_arg_s worker_args[MAX_NB_WORKER]; odph_ethaddr_t eth_dest_mac[MAX_NB_PKTIO]; /** Global barrier to synchronize main and workers */ @@ -937,14 +936,15 @@ static int print_speed_stats(int num_workers, int duration, int timeout) int main(int argc, char **argv) { - odph_odpthread_t thread_tbl[MAX_NB_WORKER]; + odph_thread_t thread_tbl[MAX_NB_WORKER]; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param[MAX_NB_WORKER]; odp_pool_t pool; odp_pool_param_t params; odp_shm_t shm; odp_instance_t instance; - odph_odpthread_params_t thr_params; odp_cpumask_t cpumask; - int cpu, i, j, nb_worker; + int i, j, nb_worker; uint8_t mac[ODPH_ETHADDR_LEN]; uint8_t *dst_mac; app_args_t *args; @@ -1101,32 +1101,25 @@ int main(int argc, char **argv) odp_barrier_init(&global->barrier, nb_worker + 1); - memset(&thr_params, 0, sizeof(thr_params)); - thr_params.start = run_worker; - thr_params.thr_type = ODP_THREAD_WORKER; - thr_params.instance = instance; + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &cpumask; - memset(thread_tbl, 0, sizeof(thread_tbl)); - cpu = odp_cpumask_first(&cpumask); for (i = 0; i < nb_worker; i++) { - struct thread_arg_s *arg; - odp_cpumask_t thr_mask; - - arg = &global->worker_args[i]; - odp_cpumask_zero(&thr_mask); - odp_cpumask_set(&thr_mask, cpu); - thr_params.arg = arg; - odph_odpthreads_create(&thread_tbl[i], &thr_mask, - &thr_params); - cpu = odp_cpumask_next(&cpumask, cpu); + odph_thread_param_init(&thr_param[i]); + thr_param[i].start = run_worker; + thr_param[i].arg = &global->worker_args[i]; + thr_param[i].thr_type = ODP_THREAD_WORKER; } + memset(thread_tbl, 0, sizeof(thread_tbl)); + odph_thread_create(thread_tbl, &thr_common, thr_param, nb_worker); + print_speed_stats(nb_worker, args->duration, PRINT_INTERVAL); odp_atomic_store_u32(&global->exit_threads, 1); /* wait for other threads to join */ - for (i = 0; i < nb_worker; i++) - odph_odpthreads_join(&thread_tbl[i]); + odph_thread_join(thread_tbl, nb_worker); /* Stop and close used pktio devices */ for (i = 0; i < args->if_count; i++) { diff --git a/example/packet/odp_pktio.c b/example/packet/odp_pktio.c index 6065bcc88..8b2bb55e3 100644 --- a/example/packet/odp_pktio.c +++ b/example/packet/odp_pktio.c @@ -342,17 +342,17 @@ static int pktio_ifburst_thread(void *arg) int main(int argc, char *argv[]) { odph_helper_options_t helper_options; - odph_odpthread_t thread_tbl[MAX_WORKERS]; + odph_thread_t thread_tbl[MAX_WORKERS]; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param[MAX_WORKERS]; odp_pool_t pool; int num_workers; int i; - int cpu; odp_cpumask_t cpumask; char cpumaskstr[ODP_CPUMASK_STR_SIZE]; odp_pool_param_t params; odp_instance_t instance; odp_init_t init_param; - odph_odpthread_params_t thr_params; odp_shm_t shm; /* Let helper collect its own arguments (e.g. --odph_proc) */ @@ -436,15 +436,11 @@ int main(int argc, char *argv[]) create_pktio(args->appl.if_names[i], pool, args->appl.mode); /* Create and init worker threads */ - memset(thread_tbl, 0, sizeof(thread_tbl)); - - memset(&thr_params, 0, sizeof(thr_params)); - thr_params.thr_type = ODP_THREAD_WORKER; - thr_params.instance = instance; + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &cpumask; - cpu = odp_cpumask_first(&cpumask); for (i = 0; i < num_workers; ++i) { - odp_cpumask_t thd_mask; int (*thr_run_func)(void *); int if_idx; @@ -457,21 +453,16 @@ int main(int argc, char *argv[]) thr_run_func = pktio_ifburst_thread; else /* APPL_MODE_PKT_QUEUE */ thr_run_func = pktio_queue_thread; - /* - * Create threads one-by-one instead of all-at-once, - * because each thread might get different arguments. - * Calls odp_thread_create(cpu) for each thread - */ - odp_cpumask_zero(&thd_mask); - odp_cpumask_set(&thd_mask, cpu); - - thr_params.start = thr_run_func; - thr_params.arg = &args->thread[i]; - - odph_odpthreads_create(&thread_tbl[i], &thd_mask, &thr_params); - cpu = odp_cpumask_next(&cpumask, cpu); + + odph_thread_param_init(&thr_param[i]); + thr_param[i].start = thr_run_func; + thr_param[i].arg = &args->thread[i]; + thr_param[i].thr_type = ODP_THREAD_WORKER; } + memset(thread_tbl, 0, sizeof(thread_tbl)); + odph_thread_create(thread_tbl, &thr_common, thr_param, num_workers); + if (args->appl.time) { odp_time_wait_ns(args->appl.time * ODP_TIME_SEC_IN_NS); @@ -487,8 +478,7 @@ int main(int argc, char *argv[]) } /* Master thread waits for other threads to exit */ - for (i = 0; i < num_workers; ++i) - odph_odpthreads_join(&thread_tbl[i]); + odph_thread_join(thread_tbl, num_workers); for (i = 0; i < args->appl.if_count; ++i) odp_pktio_close(odp_pktio_lookup(args->thread[i].pktio_dev)); diff --git a/example/simple_pipeline/odp_simple_pipeline.c b/example/simple_pipeline/odp_simple_pipeline.c index b01efee84..b8fb2d0be 100644 --- a/example/simple_pipeline/odp_simple_pipeline.c +++ b/example/simple_pipeline/odp_simple_pipeline.c @@ -853,14 +853,14 @@ int main(int argc, char **argv) stats[i] = &global->thread[i].stats; memset(thr_tbl, 0, sizeof(thr_tbl)); - memset(thr_param, 0, sizeof(thr_param)); - memset(&thr_common, 0, sizeof(thr_common)); + odph_thread_common_param_init(&thr_common); thr_common.instance = instance; /* RX thread */ thr_args = &global->thread[0]; thr_args->tx_queue = global->queue[0]; + odph_thread_param_init(&thr_param[0]); thr_param[0].start = rx_thread; thr_param[0].arg = thr_args; thr_param[0].thr_type = ODP_THREAD_WORKER; @@ -873,6 +873,7 @@ int main(int argc, char **argv) thr_args->rx_queue = global->queue[i]; thr_args->tx_queue = global->queue[i + 1]; + odph_thread_param_init(&thr_param[i]); thr_param[i].start = worker_thread; thr_param[i].arg = thr_args; thr_param[i].thr_type = ODP_THREAD_WORKER; @@ -887,6 +888,7 @@ int main(int argc, char **argv) /* TX thread */ thr_args = &global->thread[num_threads - 1]; thr_args->rx_queue = global->queue[num_workers]; + odph_thread_param_init(&thr_param[0]); thr_param[0].start = tx_thread; thr_param[0].arg = thr_args; thr_param[0].thr_type = ODP_THREAD_WORKER; diff --git a/example/switch/odp_switch.c b/example/switch/odp_switch.c index 989016fc8..f30d468f7 100644 --- a/example/switch/odp_switch.c +++ b/example/switch/odp_switch.c @@ -971,9 +971,10 @@ static void gbl_args_init(args_t *args) int main(int argc, char **argv) { odph_helper_options_t helper_options; - odph_odpthread_t thread_tbl[MAX_WORKERS]; + odph_thread_t thread_tbl[MAX_WORKERS]; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param[MAX_WORKERS]; int i, j; - int cpu; int num_workers; odp_shm_t shm; odp_cpumask_t cpumask; @@ -984,7 +985,6 @@ int main(int argc, char **argv) int if_count; odp_instance_t instance; odp_init_t init_param; - odph_odpthread_params_t thr_params; signal(SIGINT, sig_handler); @@ -1097,27 +1097,23 @@ int main(int argc, char **argv) stats = gbl_args->stats; - memset(&thr_params, 0, sizeof(thr_params)); - thr_params.thr_type = ODP_THREAD_WORKER; - thr_params.instance = instance; - thr_params.start = run_worker; + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &cpumask; /* Create worker threads */ - cpu = odp_cpumask_first(&cpumask); for (i = 0; i < num_workers; ++i) { - odp_cpumask_t thd_mask; - for (j = 0; j < MAX_PKTIOS; j++) gbl_args->thread[i].stats[j] = &stats[i][j]; - thr_params.arg = &gbl_args->thread[i]; - - odp_cpumask_zero(&thd_mask); - odp_cpumask_set(&thd_mask, cpu); - odph_odpthreads_create(&thread_tbl[i], &thd_mask, &thr_params); - cpu = odp_cpumask_next(&cpumask, cpu); + odph_thread_param_init(&thr_param[i]); + thr_param[i].start = run_worker; + thr_param[i].arg = &gbl_args->thread[i]; + thr_param[i].thr_type = ODP_THREAD_WORKER; } + odph_thread_create(thread_tbl, &thr_common, thr_param, num_workers); + /* Start packet receive and transmit */ for (i = 0; i < if_count; ++i) { odp_pktio_t pktio; @@ -1136,8 +1132,7 @@ int main(int argc, char **argv) odp_atomic_store_u32(&gbl_args->exit_threads, 1); /* Master thread waits for other threads to exit */ - for (i = 0; i < num_workers; ++i) - odph_odpthreads_join(&thread_tbl[i]); + odph_thread_join(thread_tbl, num_workers); /* Stop and close used pktio devices */ for (i = 0; i < if_count; i++) { diff --git a/example/time/time_global_test.c b/example/time/time_global_test.c index ad1bfead8..2357f25d6 100644 --- a/example/time/time_global_test.c +++ b/example/time/time_global_test.c @@ -343,8 +343,8 @@ int main(int argc, char *argv[]) goto err; } - memset(&thr_common, 0, sizeof(thr_common)); - memset(&thr_param, 0, sizeof(thr_param)); + odph_thread_common_param_init(&thr_common); + odph_thread_param_init(&thr_param); thr_param.start = run_thread; thr_param.arg = gbls; diff --git a/example/timer/odp_timer_test.c b/example/timer/odp_timer_test.c index 8bfb99d8f..b10559e6e 100644 --- a/example/timer/odp_timer_test.c +++ b/example/timer/odp_timer_test.c @@ -332,7 +332,9 @@ static int parse_args(int argc, char *argv[], test_args_t *args) int main(int argc, char *argv[]) { odph_helper_options_t helper_options; - odph_odpthread_t thread_tbl[MAX_WORKERS]; + odph_thread_t thread_tbl[MAX_WORKERS]; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; int num_workers; odp_queue_t queue; uint64_t tick, ns; @@ -344,7 +346,6 @@ int main(int argc, char *argv[]) char cpumaskstr[ODP_CPUMASK_STR_SIZE]; odp_instance_t instance; odp_init_t init_param; - odph_odpthread_params_t thr_params; odp_shm_t shm = ODP_SHM_INVALID; test_globals_t *gbls = NULL; int err = 0; @@ -509,16 +510,20 @@ int main(int argc, char *argv[]) odp_barrier_init(&gbls->test_barrier, num_workers); /* Create and launch worker threads */ - memset(&thr_params, 0, sizeof(thr_params)); - thr_params.start = run_thread; - thr_params.arg = gbls; - thr_params.thr_type = ODP_THREAD_WORKER; - thr_params.instance = instance; + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &cpumask; + thr_common.share_param = 1; - odph_odpthreads_create(thread_tbl, &cpumask, &thr_params); + odph_thread_param_init(&thr_param); + thr_param.start = run_thread; + thr_param.arg = gbls; + thr_param.thr_type = ODP_THREAD_WORKER; + + odph_thread_create(thread_tbl, &thr_common, &thr_param, num_workers); /* Wait for worker threads to exit */ - odph_odpthreads_join(thread_tbl); + odph_thread_join(thread_tbl, num_workers); /* free resources */ if (odp_queue_destroy(queue)) diff --git a/helper/cli.c b/helper/cli.c index ebc5b01e1..8dc6b66fa 100644 --- a/helper/cli.c +++ b/helper/cli.c @@ -20,8 +20,8 @@ /* Socketpair socket roles. */ enum { - SP_READ = 0, - SP_WRITE = 1, + SP_SERVER = 0, + SP_CONTROL = 1, }; #define MAX_NAME_LEN 20 @@ -35,7 +35,7 @@ typedef struct { typedef struct { volatile int cli_fd; - /* Server thread will exit if this is false. */ + /* Server will exit if this is false. */ volatile int run; /* Socketpair descriptors. */ int sp[2]; @@ -43,10 +43,9 @@ typedef struct { /* Guards cli_fd and run, which must be accessed atomically. */ odp_spinlock_t lock; odp_spinlock_t api_lock; - odph_thread_t thr_server; odp_instance_t instance; + odph_cli_param_t cli_param; struct sockaddr_in addr; - uint32_t max_user_commands; uint32_t num_user_commands; user_cmd_t user_cmd[0]; } cli_shm_t; @@ -57,6 +56,7 @@ static const odph_cli_param_t param_default = { .address = "127.0.0.1", .port = 55555, .max_user_commands = 50, + .hostname = "ODP", }; void odph_cli_param_init(odph_cli_param_t *param) @@ -99,8 +99,6 @@ int odph_cli_init(odp_instance_t instance, const odph_cli_param_t *param) memset(shm, 0, shm_size); odp_spinlock_init(&shm->lock); odp_spinlock_init(&shm->api_lock); - shm->sp[SP_READ] = -1; - shm->sp[SP_WRITE] = -1; shm->listen_fd = -1; shm->cli_fd = -1; shm->instance = instance; @@ -119,7 +117,12 @@ int odph_cli_init(odp_instance_t instance, const odph_cli_param_t *param) break; } - shm->max_user_commands = param->max_user_commands; + if (socketpair(PF_LOCAL, SOCK_STREAM, 0, shm->sp)) { + ODPH_ERR("Error: socketpair(): %s\n", strerror(errno)); + return -1; + } + + shm->cli_param = *param; return 0; } @@ -144,7 +147,7 @@ int odph_cli_register_command(const char *name, odph_cli_user_cmd_func_t func, } odp_spinlock_unlock(&shm->lock); - if (shm->num_user_commands >= shm->max_user_commands) { + if (shm->num_user_commands >= shm->cli_param.max_user_commands) { ODPH_ERR("Error: maximum number of user commands already registered\n"); goto error; } @@ -369,7 +372,7 @@ static struct cli_def *create_cli(cli_shm_t *shm) cli = cli_init(); cli_set_banner(cli, NULL); - cli_set_hostname(cli, "ODP"); + cli_set_hostname(cli, shm->cli_param.hostname); c = cli_register_command(cli, NULL, "call", NULL, PRIVILEGE_UNPRIVILEGED, MODE_EXEC, @@ -526,20 +529,48 @@ int odph_cli_log(const char *fmt, ...) return r; } -static int cli_server(void *arg ODP_UNUSED) +ODP_PRINTF_FORMAT(1, 0) +int odph_cli_log_va(const char *fmt, va_list in_args) { - cli_shm_t *shm = shm_lookup(); + int r; - if (!shm) { - ODPH_ERR("Error: shm %s not found\n", shm_name); + r = cli_log_va(ODP_LOG_PRINT, fmt, in_args); + return r; +} + +static int msg_recv(int fd) +{ + uint32_t msg; + int num = recv(fd, &msg, sizeof(msg), MSG_NOSIGNAL); + + if (num != sizeof(msg)) { + ODPH_ERR("Error: recv() = %d: %s\n", num, strerror(errno)); return -1; } + return 0; +} + +static int msg_send(int fd) +{ + uint32_t msg = 0; + int num = send(fd, &msg, sizeof(msg), MSG_DONTWAIT | MSG_NOSIGNAL); + + if (num != sizeof(msg)) { + ODPH_ERR("Error: send() = %d: %s\n", num, strerror(errno)); + return -1; + } + + return 0; +} + +static int cli_server(cli_shm_t *shm) +{ cli = create_cli(shm); while (1) { struct pollfd pfd[2] = { - { .fd = shm->sp[SP_READ], .events = POLLIN, }, + { .fd = shm->sp[SP_SERVER], .events = POLLIN, }, { .fd = shm->listen_fd, .events = POLLIN, }, }; @@ -632,10 +663,13 @@ static int cli_server(void *arg ODP_UNUSED) cli_done(cli); + if (msg_send(shm->sp[SP_SERVER])) + return -1; + return 0; } -int odph_cli_start(void) +int odph_cli_run(void) { cli_shm_t *shm = shm_lookup(); @@ -649,22 +683,13 @@ int odph_cli_start(void) if (shm->run) { odp_spinlock_unlock(&shm->lock); odp_spinlock_unlock(&shm->api_lock); - ODPH_ERR("Error: cli server has already been started\n"); + ODPH_ERR("Error: cli server is already running\n"); return -1; } shm->run = 1; shm->cli_fd = -1; odp_spinlock_unlock(&shm->lock); - shm->sp[SP_READ] = -1; - shm->sp[SP_WRITE] = -1; - shm->listen_fd = -1; - - if (socketpair(PF_LOCAL, SOCK_STREAM, 0, shm->sp)) { - ODPH_ERR("Error: socketpair(): %s\n", strerror(errno)); - goto error; - } - /* Create listening socket. */ shm->listen_fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); @@ -691,41 +716,12 @@ int odph_cli_start(void) goto error; } - /* Create server thread. */ - - odp_cpumask_t cpumask; - odph_thread_common_param_t thr_common; - odph_thread_param_t thr_param; - - if (odp_cpumask_default_control(&cpumask, 1) != 1) { - ODPH_ERR("Error: odp_cpumask_default_control() failed\n"); - goto error; - } - - memset(&thr_common, 0, sizeof(thr_common)); - thr_common.instance = shm->instance; - thr_common.cpumask = &cpumask; - - memset(&thr_param, 0, sizeof(thr_param)); - thr_param.thr_type = ODP_THREAD_CONTROL; - thr_param.start = cli_server; - - memset(&shm->thr_server, 0, sizeof(shm->thr_server)); - - if (odph_thread_create(&shm->thr_server, &thr_common, &thr_param, 1) != 1) { - ODPH_ERR("Error: odph_thread_create() failed\n"); - goto error; - } - odp_spinlock_unlock(&shm->api_lock); - return 0; + + return cli_server(shm); error: shm->run = 0; - if (shm->sp[SP_READ] >= 0) - close(shm->sp[SP_READ]); - if (shm->sp[SP_WRITE] >= 0) - close(shm->sp[SP_WRITE]); if (shm->listen_fd >= 0) close(shm->listen_fd); if (shm->cli_fd >= 0) @@ -767,22 +763,15 @@ int odph_cli_stop(void) * Send a message to the server thread in order to break it out of a * blocking poll() call. */ - int stop = 1; - int sent = send(shm->sp[SP_WRITE], &stop, - sizeof(stop), MSG_DONTWAIT | MSG_NOSIGNAL); - - if (sent != sizeof(stop)) { - ODPH_ERR("Error: send() = %d: %s\n", sent, strerror(errno)); + if (msg_send(shm->sp[SP_CONTROL])) goto error; - } - if (odph_thread_join(&shm->thr_server, 1) != 1) { - ODPH_ERR("Error: odph_thread_join() failed\n"); + /* + * Wait for the server to exit. + */ + if (msg_recv(shm->sp[SP_CONTROL])) goto error; - } - close(shm->sp[SP_READ]); - close(shm->sp[SP_WRITE]); close(shm->listen_fd); odp_spinlock_unlock(&shm->api_lock); return 0; @@ -805,6 +794,9 @@ int odph_cli_term(void) return -1; } + close(shm->sp[SP_SERVER]); + close(shm->sp[SP_CONTROL]); + if (odp_shm_free(shm_hdl)) { ODPH_ERR("Error: odp_shm_free() failed\n"); return -1; diff --git a/helper/include/odp/helper/cli.h b/helper/include/odp/helper/cli.h index ef1c24d05..982509f53 100644 --- a/helper/include/odp/helper/cli.h +++ b/helper/include/odp/helper/cli.h @@ -24,6 +24,7 @@ extern "C" { #include <odp_api.h> #include <odp/helper/ip.h> #include <stdint.h> +#include <stdarg.h> /** * @addtogroup odph_cli ODPH CLI @@ -52,6 +53,8 @@ typedef struct { uint16_t port; /** Maximum number of user defined commands. Default is 50. */ uint32_t max_user_commands; + /** Hostname to be displayed as the first part of the prompt. */ + const char *hostname; } odph_cli_param_t; /** @@ -68,7 +71,10 @@ void odph_cli_param_init(odph_cli_param_t *param); * Initialize CLI helper * * This function initializes the CLI helper. It must be called before - * odph_cli_register_command() and odph_cli_start(). + * odph_cli_register_command() and odph_cli_run(). + * + * In process mode (ODPH_PROC_MODE), this function must be called before + * creating the thread which calls odph_cli_run(). * * @param instance ODP instance * @param param CLI server parameters to use @@ -89,7 +95,7 @@ int odph_cli_init(odp_instance_t instance, const odph_cli_param_t *param); * the case they were registered in, but they may be invoked using any case. * * This function should be called after odph_cli_init() and before - * odph_cli_start(). + * odph_cli_run(). * * @param name Command name (case-insensitive) * @param func Command function @@ -102,26 +108,26 @@ int odph_cli_register_command(const char *name, odph_cli_user_cmd_func_t func, const char *help); /** - * Start CLI server + * Run CLI server * - * Upon successful return from this function, the CLI server will be - * accepting client connections. This function spawns a new thread of - * type ODP_THREAD_CONTROL using odp_cpumask_default_control(). + * When executing this function, the CLI is accepting client connections and + * running commands from a client, if one is connected. * * This function should be called after odph_cli_init() and after any - * odph_cli_register_command() calls. + * odph_cli_register_command() calls. After calling this function, + * odph_cli_stop() must be called before calling this function again. + * + * Returns only on a fatal error, or after odph_cli_stop() is called. * * @retval 0 Success * @retval <0 Failure */ -int odph_cli_start(void); +int odph_cli_run(void); /** * Stop CLI server * - * Stop accepting new client connections and disconnect currently - * connected client. This function terminates the control thread - * created in odph_cli_start(). + * Stop accepting new client connections and disconnect any connected client. * * @retval 0 Success * @retval <0 Failure @@ -144,6 +150,20 @@ int odph_cli_stop(void); int odph_cli_log(const char *fmt, ...); /** + * Print to CLI + * + * Similar to odph_cli_log(), except that this one takes its arguments as + * a va_list. + * + * @param fmt printf-style message format + * @param in_args variadic arguments + * @return On success, the number of characters printed or buffered, without + * accounting for any line feed conversions. If an error is encountered, + * a negative value is returned. + */ +int odph_cli_log_va(const char *fmt, va_list in_args); + +/** * Terminate CLI helper * * Free any resources allocated by the CLI helper. diff --git a/helper/include/odp/helper/threads.h b/helper/include/odp/helper/threads.h index 34f539f1a..d7cb0ac2a 100644 --- a/helper/include/odp/helper/threads.h +++ b/helper/include/odp/helper/threads.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019, Nokia + * Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -69,6 +69,12 @@ typedef struct { /** @deprecated ODP instance handle for odph_odpthreads_create(). */ odp_instance_t instance; + /** + * Minimum stack size in bytes. 0 = use default. Ignored by + * odph_odpthreads_create(). + */ + uint64_t stack_size; + } odph_thread_param_t; /** Helper internal thread start arguments. Used both in process and thread @@ -121,10 +127,10 @@ typedef struct { odp_mem_model_t mem_model; /**< Process or thread */ } odph_helper_options_t; -/** Legacy thread table entry */ +/** @deprecated Legacy thread table entry */ typedef odph_thread_t odph_odpthread_t; -/** Legacy thread parameters */ +/** @deprecated Legacy thread parameters */ typedef odph_thread_param_t odph_odpthread_params_t; /** Common parameters for odph_thread_create() call */ @@ -165,6 +171,19 @@ typedef struct { int sync; /** + * Synchronized thread creation timeout in nanoseconds + * + * When synchronized thread creation has been requested, waiting for the + * synchronization signal times out once the time indicated by this + * parameter has passed. + * + * If this parameter is 0, the default value is used. + * + * Default value is ODP_TIME_SEC_IN_NS. + */ + uint64_t sync_timeout; + + /** * Thread parameter sharing * * 0: Thread parameters are not shared. The thread parameter table @@ -179,6 +198,25 @@ typedef struct { } odph_thread_common_param_t; /** + * Initialize thread params + * + * Initialize an odph_thread_param_t to its default values for all fields. + * + * @param[out] param Pointer to parameter structure + */ +void odph_thread_param_init(odph_thread_param_t *param); + +/** + * Initialize thread common params + * + * Initialize an odph_thread_common_param_t to its default values for all + * fields. + * + * @param[out] param Pointer to parameter structure + */ +void odph_thread_common_param_init(odph_thread_common_param_t *param); + +/** * Create and pin threads (as Linux pthreads or processes) * * This is an updated version of odph_odpthreads_create() call. It may be called @@ -197,6 +235,9 @@ typedef struct { * with a single thread parameter table element by setting 'share_param' * parameter. * + * Use odph_thread_common_param_init() and odph_thread_param_init() to + * initialize parameters with default values. + * * Thread table must be large enough to hold 'num' elements. Also the cpumask * must contain 'num' CPUs. Threads are pinned to CPUs in order - the first * thread goes to the smallest CPU number of the mask, etc. diff --git a/helper/test/cli.c b/helper/test/cli.c index e7ed0e06f..475a99b90 100644 --- a/helper/test/cli.c +++ b/helper/test/cli.c @@ -7,6 +7,16 @@ #include <odp_api.h> #include <odp/helper/odph_api.h> +static int cli_server(void *arg ODP_UNUSED) +{ + if (odph_cli_run()) { + ODPH_ERR("odph_cli_run() failed.\n"); + exit(EXIT_FAILURE); + } + + return 0; +} + int main(int argc, char *argv[]) { odp_instance_t instance; @@ -43,16 +53,46 @@ int main(int argc, char *argv[]) exit(EXIT_FAILURE); } - if (odph_cli_start()) { - ODPH_ERR("Error: odph_cli_start() failed.\n"); + odp_cpumask_t cpumask; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; + odph_thread_t thr_server; + + if (odp_cpumask_default_control(&cpumask, 1) != 1) { + ODPH_ERR("Failed to get default CPU mask.\n"); exit(EXIT_FAILURE); } + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &cpumask; + + odph_thread_param_init(&thr_param); + thr_param.thr_type = ODP_THREAD_CONTROL; + thr_param.start = cli_server; + + memset(&thr_server, 0, sizeof(thr_server)); + + if (odph_thread_create(&thr_server, &thr_common, &thr_param, 1) != 1) { + ODPH_ERR("Failed to create server thread.\n"); + exit(EXIT_FAILURE); + } + + /* + * Wait for a bit to ensure that the server thread has time to start. + */ + odp_time_wait_ns(ODP_TIME_SEC_IN_NS / 10); + if (odph_cli_stop()) { ODPH_ERR("Error: odph_cli_stop() failed.\n"); exit(EXIT_FAILURE); } + if (odph_thread_join(&thr_server, 1) != 1) { + ODPH_ERR("Failed to join server thread.\n"); + exit(EXIT_FAILURE); + } + if (odph_cli_term()) { ODPH_ERR("Error: odph_cli_term() failed.\n"); exit(EXIT_FAILURE); diff --git a/helper/test/odpthreads.c b/helper/test/odpthreads.c index 1540ad503..aa06df188 100644 --- a/helper/test/odpthreads.c +++ b/helper/test/odpthreads.c @@ -1,4 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -12,6 +13,11 @@ #include <unistd.h> #include <stdlib.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <errno.h> +#include <string.h> +#include <inttypes.h> #include <odp_api.h> #include <odp/helper/odph_api.h> @@ -62,14 +68,18 @@ static int worker_fn(void *arg ODP_UNUSED) int main(int argc, char *argv[]) { odph_helper_options_t helper_options; - odph_odpthread_params_t thr_params; - odph_odpthread_t thread_tbl[NUMBER_WORKERS]; + odph_thread_t thread_tbl[NUMBER_WORKERS]; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; odp_cpumask_t cpu_mask; odp_init_t init_param; int num_workers; int cpu, affinity; int ret; char cpumaskstr[ODP_CPUMASK_STR_SIZE]; + struct rlimit rlimit; + pthread_attr_t attr; + size_t stack_size; /* Let helper collect its own arguments (e.g. --odph_proc) */ argc = odph_parse_options(argc, argv); @@ -143,18 +153,56 @@ int main(int argc, char *argv[]) printf("new cpu mask: %s\n", cpumaskstr); printf("new num worker threads: %i\n\n", num_workers); - memset(&thr_params, 0, sizeof(thr_params)); - thr_params.start = worker_fn; - thr_params.arg = NULL; - thr_params.thr_type = ODP_THREAD_WORKER; - thr_params.instance = odp_instance; + odph_thread_common_param_init(&thr_common); + thr_common.instance = odp_instance; + thr_common.cpumask = &cpu_mask; + thr_common.share_param = 1; - odph_odpthreads_create(&thread_tbl[0], &cpu_mask, &thr_params); + odph_thread_param_init(&thr_param); + thr_param.start = worker_fn; + thr_param.arg = NULL; + thr_param.thr_type = ODP_THREAD_WORKER; - ret = odph_odpthreads_join(thread_tbl); + odph_thread_create(thread_tbl, &thr_common, &thr_param, num_workers); + + ret = odph_thread_join(thread_tbl, num_workers); if (ret < 0) exit(EXIT_FAILURE); + /* Test threads with non-default stack size and sync timeout. */ + + pthread_attr_init(&attr); + + if (pthread_attr_getstacksize(&attr, &stack_size)) { + ODPH_ERR("pthread_attr_getstacksize() failed\n"); + return -1; + } + + printf("\n"); + printf("pthread default stack size: %zu\n", stack_size); + + if (getrlimit(RLIMIT_STACK, &rlimit)) { + ODPH_ERR("getrlimit() failed: %s\n", strerror(errno)); + return -1; + } + + printf("stack size soft limit (rlim_cur): %lu\n", rlimit.rlim_cur); + + if (rlimit.rlim_cur < stack_size) + stack_size = rlimit.rlim_cur; + + thr_param.stack_size = stack_size - ODP_PAGE_SIZE; + printf("use stack size: %" PRIu64 "\n", thr_param.stack_size); + thr_common.sync_timeout = 5 * ODP_TIME_SEC_IN_NS; + printf("use sync timeout: %" PRIu64 "\n", thr_common.sync_timeout); + printf("\n"); + + if (odph_thread_create(thread_tbl, &thr_common, &thr_param, num_workers) != num_workers) + exit(EXIT_FAILURE); + + if (odph_thread_join(thread_tbl, num_workers) != num_workers) + exit(EXIT_FAILURE); + return 0; } diff --git a/helper/threads.c b/helper/threads.c index f8244a89b..9326e12e6 100644 --- a/helper/threads.c +++ b/helper/threads.c @@ -15,6 +15,10 @@ #include <sys/wait.h> #include <sys/prctl.h> #include <sys/syscall.h> +#include <errno.h> +#include <limits.h> +#include <sys/time.h> +#include <sys/resource.h> #include <odp_api.h> #include <odp/helper/threads.h> @@ -81,7 +85,7 @@ static void *run_thread(void *arg) /* * Create a single linux process */ -static int create_process(odph_thread_t *thread, int cpu) +static int create_process(odph_thread_t *thread, int cpu, uint64_t stack_size) { cpu_set_t cpu_set; pid_t pid; @@ -118,6 +122,22 @@ static int create_process(odph_thread_t *thread, int cpu) return -2; } + if (stack_size) { + struct rlimit rlimit; + + if (getrlimit(RLIMIT_STACK, &rlimit)) { + ODPH_ERR("getrlimit() failed: %s\n", strerror(errno)); + return -3; + } + + rlimit.rlim_cur = stack_size; + + if (setrlimit(RLIMIT_STACK, &rlimit)) { + ODPH_ERR("setrlimit() failed: %s\n", strerror(errno)); + return -4; + } + } + run_thread(&thread->start_args); return 0; /* never reached */ @@ -160,7 +180,7 @@ static int wait_process(odph_thread_t *thread) /* * Create a single linux pthread */ -static int create_pthread(odph_thread_t *thread, int cpu) +static int create_pthread(odph_thread_t *thread, int cpu, uint64_t stack_size) { int ret; cpu_set_t cpu_set; @@ -175,6 +195,24 @@ static int create_pthread(odph_thread_t *thread, int cpu) pthread_attr_setaffinity_np(&thread->thread.attr, sizeof(cpu_set_t), &cpu_set); + if (stack_size) { + /* + * Round up to page size. "On some systems, + * pthread_attr_setstacksize() can fail with the error EINVAL if + * stacksize is not a multiple of the system page size." (man + * page) + */ + stack_size = (stack_size + ODP_PAGE_SIZE - 1) & ~(ODP_PAGE_SIZE - 1); + + if (stack_size < PTHREAD_STACK_MIN) + stack_size = PTHREAD_STACK_MIN; + + if (pthread_attr_setstacksize(&thread->thread.attr, stack_size)) { + ODPH_ERR("pthread_attr_setstacksize() failed\n"); + return -1; + } + } + thread->start_args.mem_model = ODP_MEM_MODEL_THREAD; ret = pthread_create(&thread->thread.thread_id, @@ -224,6 +262,17 @@ static int wait_pthread(odph_thread_t *thread) return 0; } +void odph_thread_param_init(odph_thread_param_t *param) +{ + memset(param, 0, sizeof(*param)); +} + +void odph_thread_common_param_init(odph_thread_common_param_t *param) +{ + memset(param, 0, sizeof(*param)); + param->sync_timeout = ODP_TIME_SEC_IN_NS; +} + int odph_thread_create(odph_thread_t thread[], const odph_thread_common_param_t *param, const odph_thread_param_t thr_param[], @@ -272,10 +321,10 @@ int odph_thread_create(odph_thread_t thread[], odp_atomic_init_u32(&start_args->status, NOT_STARTED); if (use_pthread) { - if (create_pthread(&thread[i], cpu)) + if (create_pthread(&thread[i], cpu, start_args->thr_params.stack_size)) break; } else { - if (create_process(&thread[i], cpu)) + if (create_process(&thread[i], cpu, start_args->thr_params.stack_size)) break; } @@ -286,6 +335,10 @@ int odph_thread_create(odph_thread_t thread[], uint32_t status; int timeout = 0; odp_atomic_u32_t *atomic = &start_args->status; + uint64_t timeout_ns = param->sync_timeout; + + if (!timeout_ns) + timeout_ns = ODP_TIME_SEC_IN_NS; t1 = odp_time_local(); @@ -293,7 +346,7 @@ int odph_thread_create(odph_thread_t thread[], odp_cpu_pause(); t2 = odp_time_local(); diff_ns = odp_time_diff_ns(t2, t1); - timeout = diff_ns > ODP_TIME_SEC_IN_NS; + timeout = diff_ns > timeout_ns; status = odp_atomic_load_acq_u32(atomic); } while (status != INIT_DONE && timeout == 0); @@ -375,10 +428,10 @@ int odph_odpthreads_create(odph_odpthread_t *thread_tbl, start_args->instance = thr_params->instance; if (helper_options.mem_model == ODP_MEM_MODEL_THREAD) { - if (create_pthread(&thread_tbl[i], cpu)) + if (create_pthread(&thread_tbl[i], cpu, 0)) break; } else { - if (create_process(&thread_tbl[i], cpu)) + if (create_process(&thread_tbl[i], cpu, 0)) break; } diff --git a/include/Makefile.am b/include/Makefile.am index 55a21a539..de9d898b6 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -25,7 +25,6 @@ odpapiinclude_HEADERS = \ odp/api/deprecated.h \ odp/api/errno.h \ odp/api/event.h \ - odp/api/feature.h \ odp/api/hash.h \ odp/api/hints.h \ odp/api/init.h \ @@ -53,7 +52,6 @@ odpapiinclude_HEADERS = \ odp/api/stash.h \ odp/api/std.h \ odp/api/std_types.h \ - odp/api/support.h \ odp/api/sync.h \ odp/api/system_info.h \ odp/api/thread.h \ @@ -81,7 +79,6 @@ odpapispecinclude_HEADERS = \ odp/api/spec/debug.h \ odp/api/spec/errno.h \ odp/api/spec/event.h \ - odp/api/spec/feature.h \ odp/api/spec/hash.h \ odp/api/spec/hints.h \ odp/api/spec/init.h \ @@ -110,7 +107,6 @@ odpapispecinclude_HEADERS = \ odp/api/spec/stash.h \ odp/api/spec/std.h \ odp/api/spec/std_types.h \ - odp/api/spec/support.h \ odp/api/spec/sync.h \ odp/api/spec/system_info.h \ odp/api/spec/thread.h \ diff --git a/include/odp/api/abi-default/cpu.h b/include/odp/api/abi-default/cpu.h index a9b9f2c29..24e5351ad 100644 --- a/include/odp/api/abi-default/cpu.h +++ b/include/odp/api/abi-default/cpu.h @@ -15,12 +15,6 @@ extern "C" { #define ODP_CACHE_LINE_SIZE 64 #endif -#ifdef _ODP_NEED_GENERIC_CPU_PAUSE -static inline void odp_cpu_pause(void) -{ -} -#endif - #ifdef __cplusplus } #endif diff --git a/include/odp/api/feature.h b/include/odp/api/feature.h deleted file mode 100644 index f8f62b666..000000000 --- a/include/odp/api/feature.h +++ /dev/null @@ -1,26 +0,0 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -/** - * @file - * - * ODP features. - */ - -#ifndef ODP_API_FEATURE_H_ -#define ODP_API_FEATURE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include <odp/api/spec/feature.h> - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/include/odp/api/spec/classification.h b/include/odp/api/spec/classification.h index f3cfea78b..ec15c8962 100644 --- a/include/odp/api/spec/classification.h +++ b/include/odp/api/spec/classification.h @@ -20,7 +20,7 @@ extern "C" { #endif #include <odp/api/packet_io.h> -#include <odp/api/support.h> +#include <odp/api/std_types.h> #include <odp/api/threshold.h> /** @defgroup odp_classification ODP CLASSIFICATION * Packet input classification. diff --git a/include/odp/api/spec/comp.h b/include/odp/api/spec/comp.h index a09bd7254..7f64cca5d 100644 --- a/include/odp/api/spec/comp.h +++ b/include/odp/api/spec/comp.h @@ -14,7 +14,7 @@ #define ODP_API_SPEC_COMP_H_ #include <odp/visibility_begin.h> -#include <odp/api/support.h> +#include <odp/api/std_types.h> #include <odp/api/packet.h> #ifdef __cplusplus diff --git a/include/odp/api/spec/crypto.h b/include/odp/api/spec/crypto.h index c5c3f0936..bfda0bcee 100644 --- a/include/odp/api/spec/crypto.h +++ b/include/odp/api/spec/crypto.h @@ -15,7 +15,7 @@ #include <odp/visibility_begin.h> #include <odp/api/deprecated.h> -#include <odp/api/support.h> +#include <odp/api/std_types.h> #ifdef __cplusplus extern "C" { diff --git a/include/odp/api/spec/feature.h b/include/odp/api/spec/feature.h deleted file mode 100644 index ccb5cf535..000000000 --- a/include/odp/api/spec/feature.h +++ /dev/null @@ -1,76 +0,0 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -/** - * @file - * - * ODP features. - * Define various ODP feature sets that can be referenced by other - * components. - */ - -#ifndef ODP_API_SPEC_FEATURE_H_ -#define ODP_API_SPEC_FEATURE_H_ -#include <odp/visibility_begin.h> - -#ifdef __cplusplus -extern "C" { -#endif - -#include <odp/api/std_types.h> - -/** @defgroup odp_features ODP_FEATURE - * List of ODP features. - * @{ - */ - -/** Definition of ODP features */ -typedef union odp_feature_t { - /** All features */ - uint32_t all_feat; - - /** Individual feature bits */ - struct { - /** Classifier APIs, e.g., odp_cls_xxx(), odp_cos_xxx() */ - uint32_t cls:1; - - /** Compression APIs, e.g., odp_comp_xxx() */ - uint32_t compress:1; - - /** Crypto APIs, e.g., odp_crypto_xxx() */ - uint32_t crypto:1; - - /** IPsec APIs, e.g., odp_ipsec_xxx() */ - uint32_t ipsec:1; - - /** Scheduler APIs, e.g., odp_schedule_xxx() */ - uint32_t schedule:1; - - /** Stash APIs, e.g., odp_stash_xxx() */ - uint32_t stash:1; - - /** Time APIs, e.g., odp_time_xxx() */ - uint32_t time:1; - - /** Timer APIs, e.g., odp_timer_xxx(), odp_timeout_xxx() */ - uint32_t timer:1; - - /** Traffic Manager APIs, e.g., odp_tm_xxx() */ - uint32_t tm:1; - } feat; - -} odp_feature_t; - -/** - * @} - */ - -#ifdef __cplusplus -} -#endif - -#include <odp/visibility_end.h> -#endif diff --git a/include/odp/api/spec/init.h b/include/odp/api/spec/init.h index d52ac50b6..67e616fce 100644 --- a/include/odp/api/spec/init.h +++ b/include/odp/api/spec/init.h @@ -19,7 +19,6 @@ extern "C" { #include <odp/api/std_types.h> #include <odp/api/hints.h> -#include <odp/api/feature.h> #include <odp/api/spec/thread_types.h> #include <odp/api/cpumask.h> diff --git a/include/odp/api/spec/ipsec.h b/include/odp/api/spec/ipsec.h index 3441d83c9..6e28e7fb4 100644 --- a/include/odp/api/spec/ipsec.h +++ b/include/odp/api/spec/ipsec.h @@ -20,7 +20,7 @@ extern "C" { #endif #include <odp/api/crypto.h> -#include <odp/api/support.h> +#include <odp/api/std_types.h> #include <odp/api/packet_io.h> #include <odp/api/protocols.h> #include <odp/api/classification.h> @@ -338,6 +338,28 @@ typedef struct odp_ipsec_capability_t { */ uint32_t max_cls_cos; + /** + * Scheduled queue support + * + * 0: Scheduled queues are not supported either as IPsec SA destination + * queues or as IPsec default queue + * 1: Scheduled queues are supported as both IPsec SA destination queues + * and IPsec default queue + * @see odp_ipsec_sa_param_t + */ + odp_bool_t queue_type_sched; + + /** + * Plain queue support + * + * 0: Plain queues are not supported either as IPsec SA destination + * queues or as IPsec default queue + * 1: Plain queues are supported as both IPsec SA destination queues and + * IPsec default queue + * @see odp_ipsec_sa_param_t + */ + odp_bool_t queue_type_plain; + /** Maximum number of different destination queues. The same queue may * be used for many SAs. */ uint32_t max_queues; @@ -1532,6 +1554,17 @@ typedef struct odp_ipsec_packet_result_t { uint32_t len; } outer_hdr; + /** Total IP length of the original ESP or AH packet before IPsec + * decapsulation. This is valid only for inbound inline and async + * processed packets. Zero value means that the length information + * is not available. + * + * If the result packet was reassembled from multiple IPsec + * protected packets, this is the sum of the lengths of all the + * involved IPsec packets. + */ + uint32_t orig_ip_len; + } odp_ipsec_packet_result_t; /** @@ -1591,9 +1624,10 @@ typedef struct odp_ipsec_status_t { * packets consumed and outputs a new packet handle for each outputted packet. * Outputted packets contain IPSEC result metadata (odp_ipsec_packet_result_t), * which should be checked for transformation errors, etc. Outputted packets - * with error status have not been transformed but the original packet is - * returned. The operation does not modify packets that it does not consume. - * It cannot consume all input packets if 'num_out' is smaller than 'num_in'. + * with error status have undefined content, except that in case of sa_lookup + * error the original input packet data is returned. The operation does not + * modify packets that it does not consume. It cannot consume all input + * packets if 'num_out' is smaller than 'num_in'. * * Packet context pointer and user area content are copied from input to output * packets. Output packets are allocated from the same pool(s) as input packets. @@ -1675,9 +1709,10 @@ int odp_ipsec_in(const odp_packet_t pkt_in[], int num_in, * packets consumed and outputs a new packet handle for each outputted packet. * Outputted packets contain IPSEC result metadata (odp_ipsec_packet_result_t), * which should be checked for transformation errors, etc. Outputted packets - * with error status have not been transformed but the original packet is - * returned. The operation does not modify packets that it does not consume. - * It cannot consume all input packets if 'num_out' is smaller than 'num_in'. + * with error status have undefined content, except that in case of MTU error + * the original input packet data is returned. The operation does not modify + * packets that it does not consume. It cannot consume all input packets if + * 'num_out' is smaller than 'num_in'. * * Packet context pointer and user area content are copied from input to output * packets. Output packets are allocated from the same pool(s) as input packets. diff --git a/include/odp/api/spec/packet.h b/include/odp/api/spec/packet.h index 2285a857d..e8f3cbc4a 100644 --- a/include/odp/api/spec/packet.h +++ b/include/odp/api/spec/packet.h @@ -180,6 +180,20 @@ void odp_packet_to_event_multi(const odp_packet_t pkt[], odp_event_t ev[], int num); /** + * Get information about successful reassembly offload that has happened + * + * This function may be called only if the reassembly status of a packet + * is ODP_PACKET_REASS_COMPLETE. + * + * @param pkt Completely reassembled packet. + * @param[out] info Pointer to the info structure to be filled + * + * @retval 0 on success + * @retval <0 on failure + */ +int odp_packet_reass_info(odp_packet_t pkt, odp_packet_reass_info_t *info); + +/** * Get partial reassembly state from a packet * * In case of incomplete reassembly, a packet carries information on diff --git a/include/odp/api/spec/packet_types.h b/include/odp/api/spec/packet_types.h index 5549f03aa..52b0e22b6 100644 --- a/include/odp/api/spec/packet_types.h +++ b/include/odp/api/spec/packet_types.h @@ -286,6 +286,14 @@ typedef enum odp_packet_reass_status_t { } odp_packet_reass_status_t; /** + * Information about a completed reassembly + */ +typedef struct odp_packet_reass_info_t { + /** Number of fragments reassembled */ + uint16_t num_frags; +} odp_packet_reass_info_t; + +/** * Result from odp_packet_reass_partial_state() */ typedef struct odp_packet_reass_partial_state_t { diff --git a/include/odp/api/spec/schedule_types.h b/include/odp/api/spec/schedule_types.h index 90146585f..8421b4107 100644 --- a/include/odp/api/spec/schedule_types.h +++ b/include/odp/api/spec/schedule_types.h @@ -14,7 +14,7 @@ #define ODP_API_SPEC_SCHEDULE_TYPES_H_ #include <odp/visibility_begin.h> -#include <odp/api/support.h> +#include <odp/api/std_types.h> #ifdef __cplusplus extern "C" { diff --git a/include/odp/api/spec/std_types.h b/include/odp/api/spec/std_types.h index 5dc350a24..41f436065 100644 --- a/include/odp/api/spec/std_types.h +++ b/include/odp/api/spec/std_types.h @@ -9,7 +9,7 @@ /** * @file * - * Standard C language types and definitions for ODP. + * Common types and definitions for ODP API files. * */ @@ -96,6 +96,66 @@ typedef struct odp_fract_u64_t { } odp_fract_u64_t; /** + * ODP support + * + * Support levels are specified in the relative order, where ODP_SUPPORT_NO is + * the lowest level. E.g. if the examined support level is greater than + * ODP_SUPPORT_NO, the feature is supported in some form. + */ +typedef enum odp_support_t { + /** + * Feature is not supported + */ + ODP_SUPPORT_NO = 0, + /** + * Feature is supported + */ + ODP_SUPPORT_YES, + /** + * Feature is supported and preferred + */ + ODP_SUPPORT_PREFERRED + +} odp_support_t; + +/** Definition of ODP features */ +typedef union odp_feature_t { + /** All features */ + uint32_t all_feat; + + /** Individual feature bits */ + struct { + /** Classifier APIs, e.g., odp_cls_xxx(), odp_cos_xxx() */ + uint32_t cls:1; + + /** Compression APIs, e.g., odp_comp_xxx() */ + uint32_t compress:1; + + /** Crypto APIs, e.g., odp_crypto_xxx() */ + uint32_t crypto:1; + + /** IPsec APIs, e.g., odp_ipsec_xxx() */ + uint32_t ipsec:1; + + /** Scheduler APIs, e.g., odp_schedule_xxx() */ + uint32_t schedule:1; + + /** Stash APIs, e.g., odp_stash_xxx() */ + uint32_t stash:1; + + /** Time APIs, e.g., odp_time_xxx() */ + uint32_t time:1; + + /** Timer APIs, e.g., odp_timer_xxx(), odp_timeout_xxx() */ + uint32_t timer:1; + + /** Traffic Manager APIs, e.g., odp_tm_xxx() */ + uint32_t tm:1; + } feat; + +} odp_feature_t; + +/** * @} */ diff --git a/include/odp/api/spec/support.h b/include/odp/api/spec/support.h deleted file mode 100644 index 10611219d..000000000 --- a/include/odp/api/spec/support.h +++ /dev/null @@ -1,57 +0,0 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -/** - * @file - * - * ODP support API - */ - -#ifndef ODP_API_SPEC_SUPPORT_H_ -#define ODP_API_SPEC_SUPPORT_H_ -#include <odp/visibility_begin.h> - -#ifdef __cplusplus -extern "C" { -#endif - -/** @defgroup odp_support ODP SUPPORT - * Feature support levels. - * @{ - */ - -/** - * ODP support support - * - * Support levels are specified in the relative order, where ODP_SUPPORT_NO is - * the lowest level. E.g. if the examined support level is greater than - * ODP_SUPPORT_NO, the feature is supported in some form. - */ -typedef enum odp_support_t { - /** - * Feature is not supported - */ - ODP_SUPPORT_NO = 0, - /** - * Feature is supported - */ - ODP_SUPPORT_YES, - /** - * Feature is supported and preferred - */ - ODP_SUPPORT_PREFERRED -} odp_support_t; - -/** - * @} - */ - -#ifdef __cplusplus -} -#endif - -#include <odp/visibility_end.h> -#endif diff --git a/include/odp/api/support.h b/include/odp/api/support.h deleted file mode 100644 index a6a3ef5a0..000000000 --- a/include/odp/api/support.h +++ /dev/null @@ -1,26 +0,0 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -/** - * @file - * - * ODP support API - platform specific header - */ - -#ifndef ODP_API_SUPPORT_H_ -#define ODP_API_SUPPORT_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include <odp/api/spec/support.h> - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/include/odp/arch/arm32-linux/odp/api/abi/cpu.h b/include/odp/arch/arm32-linux/odp/api/abi/cpu.h index 0ab5b8e14..d7485c090 100644 --- a/include/odp/arch/arm32-linux/odp/api/abi/cpu.h +++ b/include/odp/arch/arm32-linux/odp/api/abi/cpu.h @@ -13,16 +13,6 @@ extern "C" { #define ODP_CACHE_LINE_SIZE 64 -static inline void odp_cpu_pause(void) -{ - /* YIELD hints the CPU to switch to another thread if possible - * and executes as a NOP otherwise. - * ISB flushes the pipeline, then restarts. This is guaranteed to - * stall the CPU a number of cycles. - */ - __asm volatile("isb" ::: "memory"); -} - #ifdef __cplusplus } #endif diff --git a/include/odp/arch/arm64-linux/odp/api/abi/cpu.h b/include/odp/arch/arm64-linux/odp/api/abi/cpu.h index 0ab5b8e14..d7485c090 100644 --- a/include/odp/arch/arm64-linux/odp/api/abi/cpu.h +++ b/include/odp/arch/arm64-linux/odp/api/abi/cpu.h @@ -13,16 +13,6 @@ extern "C" { #define ODP_CACHE_LINE_SIZE 64 -static inline void odp_cpu_pause(void) -{ - /* YIELD hints the CPU to switch to another thread if possible - * and executes as a NOP otherwise. - * ISB flushes the pipeline, then restarts. This is guaranteed to - * stall the CPU a number of cycles. - */ - __asm volatile("isb" ::: "memory"); -} - #ifdef __cplusplus } #endif diff --git a/include/odp/arch/default-linux/odp/api/abi/cpu.h b/include/odp/arch/default-linux/odp/api/abi/cpu.h index bddc7627c..d7485c090 100644 --- a/include/odp/arch/default-linux/odp/api/abi/cpu.h +++ b/include/odp/arch/default-linux/odp/api/abi/cpu.h @@ -13,10 +13,6 @@ extern "C" { #define ODP_CACHE_LINE_SIZE 64 -static inline void odp_cpu_pause(void) -{ -} - #ifdef __cplusplus } #endif diff --git a/include/odp/arch/mips64-linux/odp/api/abi/cpu.h b/include/odp/arch/mips64-linux/odp/api/abi/cpu.h index 450bd0928..6e2573a74 100644 --- a/include/odp/arch/mips64-linux/odp/api/abi/cpu.h +++ b/include/odp/arch/mips64-linux/odp/api/abi/cpu.h @@ -17,14 +17,6 @@ extern "C" { #error Please add support for your arch in abi/cpu.h #endif -static inline void odp_cpu_pause(void) -{ - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); -} - #ifdef __cplusplus } #endif diff --git a/include/odp/arch/power64-linux/odp/api/abi/cpu.h b/include/odp/arch/power64-linux/odp/api/abi/cpu.h index 9e3338d60..42a5dd22c 100644 --- a/include/odp/arch/power64-linux/odp/api/abi/cpu.h +++ b/include/odp/arch/power64-linux/odp/api/abi/cpu.h @@ -1,9 +1,21 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#define _ODP_NEED_GENERIC_CPU_PAUSE +#ifndef ODP_API_ABI_CPU_H_ +#define ODP_API_ABI_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + #define ODP_CACHE_LINE_SIZE 128 -#include <odp/api/abi-default/cpu.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/arch/x86_64-linux/odp/api/abi/cpu.h b/include/odp/arch/x86_64-linux/odp/api/abi/cpu.h index 69b82eb05..d7485c090 100644 --- a/include/odp/arch/x86_64-linux/odp/api/abi/cpu.h +++ b/include/odp/arch/x86_64-linux/odp/api/abi/cpu.h @@ -4,8 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#ifndef ODP_API_ABI_CPU_ARCH_H_ -#define ODP_API_ABI_CPU_ARCH_H_ +#ifndef ODP_API_ABI_CPU_H_ +#define ODP_API_ABI_CPU_H_ #ifdef __cplusplus extern "C" { diff --git a/include/odp_api.h b/include/odp_api.h index 84530d2ab..8b129f939 100644 --- a/include/odp_api.h +++ b/include/odp_api.h @@ -33,7 +33,6 @@ extern "C" { #include <odp/api/barrier.h> #include <odp/api/spinlock.h> #include <odp/api/atomic.h> -#include <odp/api/feature.h> #include <odp/api/init.h> #include <odp/api/system_info.h> #include <odp/api/thread.h> @@ -61,7 +60,6 @@ extern "C" { #include <odp/api/spinlock_recursive.h> #include <odp/api/rwlock_recursive.h> #include <odp/api/std.h> -#include <odp/api/support.h> #include <odp/api/ipsec.h> #include <odp/api/stash.h> #include <odp/api/reassembly.h> diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 61b61f6d6..279aed0c1 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -242,12 +242,13 @@ __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ arch/default/odp_hash_crc32.c \ arch/arm/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/arm/odp/api/abi/cpu_inlines.h \ arch/arm/odp/api/abi/cpu.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ @@ -264,14 +265,15 @@ __LIB__libodp_dpdk_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/aarch64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/aarch64/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu_time.h \ arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/aarch64/odp/api/abi/atomic_inlines.h \ arch/aarch64/odp/api/abi/atomic.h \ - arch/aarch64/odp/api/abi/cpu.h + arch/default/odp/api/abi/cpu_generic.h \ + arch/aarch64/odp/api/abi/cpu_inlines.h \ + arch/aarch64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ @@ -284,12 +286,13 @@ __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -298,16 +301,16 @@ noinst_HEADERS += arch/default/odp_atomic.h \ endif if ARCH_IS_MIPS64 __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ - arch/mips64/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/mips64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/mips64/odp/api/abi/cpu_inlines.h \ arch/mips64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -320,12 +323,13 @@ __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/powerpc/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/default/odp/api/abi/cpu_inlines.h \ arch/powerpc/odp/api/abi/cpu.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -339,13 +343,13 @@ __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/x86/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_inlines.h \ - arch/x86/odp/api/abi/cpu_rdtsc.h \ +odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_rdtsc.h \ arch/x86/odp/api/abi/cpu_time.h \ arch/x86/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/x86/odp/api/abi/cpu_inlines.h \ arch/x86/odp/api/abi/cpu.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h b/platform/linux-dpdk/include/odp_buffer_internal.h index 37e5d1b4f..e47030ea7 100644 --- a/platform/linux-dpdk/include/odp_buffer_internal.h +++ b/platform/linux-dpdk/include/odp_buffer_internal.h @@ -29,7 +29,6 @@ extern "C" { #include <sys/types.h> #include <odp/api/event.h> #include <odp_forward_typedefs_internal.h> -#include <odp_schedule_if.h> #include <stddef.h> /* DPDK */ diff --git a/platform/linux-dpdk/include/odp_config_internal.h b/platform/linux-dpdk/include/odp_config_internal.h index 73d2304c9..6618d413d 100644 --- a/platform/linux-dpdk/include/odp_config_internal.h +++ b/platform/linux-dpdk/include/odp_config_internal.h @@ -124,11 +124,18 @@ extern "C" { CONFIG_PACKET_HEADROOM - \ CONFIG_PACKET_TAILROOM) -/* Maximum number of shared memory blocks. +/* + * Number of shared memory blocks reserved for implementation internal use. + */ +#define CONFIG_INTERNAL_SHM_BLOCKS 20 + +/* + * Maximum number of shared memory blocks. * - * This the number of separate SHM areas that can be reserved concurrently + * This is the number of separate SHM blocks that an application can reserve + * concurrently. */ -#define ODP_CONFIG_SHM_BLOCKS (ODP_CONFIG_POOLS + 48) +#define CONFIG_SHM_BLOCKS 64 /* * Maximum event burst size diff --git a/platform/linux-dpdk/include/odp_eventdev_internal.h b/platform/linux-dpdk/include/odp_eventdev_internal.h index 496a2238f..9e1083fd5 100644 --- a/platform/linux-dpdk/include/odp_eventdev_internal.h +++ b/platform/linux-dpdk/include/odp_eventdev_internal.h @@ -36,6 +36,8 @@ extern "C" { #include <stdint.h> +#define _ODP_SCHED_ID_EVENTDEV (_ODP_SCHED_ID_SCALABLE + 1) + #define RX_ADAPTER_INIT 0 #define RX_ADAPTER_STOPPED 1 #define RX_ADAPTER_RUNNING 2 diff --git a/platform/linux-dpdk/include/odp_queue_basic_internal.h b/platform/linux-dpdk/include/odp_queue_basic_internal.h index c3ddaf334..1e542d973 100644 --- a/platform/linux-dpdk/include/odp_queue_basic_internal.h +++ b/platform/linux-dpdk/include/odp_queue_basic_internal.h @@ -116,6 +116,9 @@ int _odp_sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int num, int update_status); int _odp_sched_queue_empty(uint32_t queue_index); +/* Functions by schedulers */ +int _odp_sched_basic_get_spread(uint32_t queue_index); + #ifdef __cplusplus } #endif diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 8ed561534..13e646230 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -64,8 +64,11 @@ typedef struct crypto_session_entry_s { struct rte_cryptodev_sym_session *rte_session; struct rte_crypto_sym_xform cipher_xform; struct rte_crypto_sym_xform auth_xform; + struct { + unsigned int cdev_qpairs_shared:1; + unsigned int chained_bufs_ok:1; + } flags; uint16_t cdev_nb_qpairs; - odp_bool_t cdev_qpairs_shared; uint8_t cdev_id; uint8_t cipher_iv_data[MAX_IV_LENGTH]; uint8_t auth_iv_data[MAX_IV_LENGTH]; @@ -312,7 +315,7 @@ int _odp_crypto_init_global(void) mem_size += (MAX_SESSIONS * sizeof(crypto_session_entry_t)); /* Allocate our globally shared memory */ - shm = odp_shm_reserve("_odp_crypto_glb", mem_size, + shm = odp_shm_reserve("_odp_crypto_global", mem_size, ODP_CACHE_LINE_SIZE, 0); if (shm != ODP_SHM_INVALID) { global = odp_shm_addr(shm); @@ -1260,6 +1263,34 @@ check_finish: return -1; } +static int chained_bufs_ok(const odp_crypto_session_param_t *param, + uint8_t cdev_id) +{ + struct rte_cryptodev_info dev_info; + int chained_bufs_ok; + + rte_cryptodev_info_get(cdev_id, &dev_info); + chained_bufs_ok = !!(dev_info.feature_flags & RTE_CRYPTODEV_FF_IN_PLACE_SGL); + + /* + * Some crypto devices do not support chained buffers with all + * algorithms despite advertizing SG support in feature flags. + */ + + if (dev_info.driver_name && + !strcmp(dev_info.driver_name, "crypto_aesni_gcm") && + param->auth_alg == ODP_AUTH_ALG_AES_GMAC) + chained_bufs_ok = 0; + + if (dev_info.driver_name && + !strcmp(dev_info.driver_name, "crypto_openssl") && + (param->cipher_alg == ODP_CIPHER_ALG_AES_GCM || + param->auth_alg == ODP_AUTH_ALG_AES_GMAC)) + chained_bufs_ok = 0; + + return chained_bufs_ok; +} + #if RTE_VERSION < RTE_VERSION_NUM(19, 8, 0, 0) static int crypto_init_key(uint8_t **data, uint16_t *length, odp_crypto_key_t *key, const char *type) @@ -1482,6 +1513,7 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, param->auth_alg == ODP_AUTH_ALG_NULL) { rte_session = NULL; cdev_id = ~0; + session->flags.chained_bufs_ok = 1; session->cdev_nb_qpairs = 0; goto out_null; } else if (param->cipher_alg == ODP_CIPHER_ALG_NULL) { @@ -1524,8 +1556,12 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, goto err; } + session->flags.chained_bufs_ok = chained_bufs_ok(param, cdev_id); session->cdev_nb_qpairs = global->enabled_crypto_dev_qpairs[cdev_id]; - session->cdev_qpairs_shared = global->enabled_crypto_dev_qpairs_shared[cdev_id]; + if (global->enabled_crypto_dev_qpairs_shared[cdev_id]) + session->flags.cdev_qpairs_shared = 1; + else + session->flags.cdev_qpairs_shared = 0; out_null: session->rte_session = rte_session; session->cdev_id = cdev_id; @@ -1840,6 +1876,50 @@ static void crypto_fill_sym_param(crypto_session_entry_t *session, op->sym->auth.data.length = param->auth_range.length; } +/* + * Attempt to change a multi segment packet to a single segment packet by + * reducing the headroom. Shift packet data toward the start of the first + * segment and trim the tail, hopefully getting rid of the tail segment. + * + * This fails if the packet data does not fit in the first segment with + * the new headroom. A temporary copy to a bigger buffer would be needed + * in that case. + * + * Do nothing for single segment packets. + * + * We assume that odp_crypto_operation() makes no promise to not shift + * packet data within the packet. If that is not the case, the shifting + * done here needs to be undone after the crypto operation. + * + */ +static int linearize_pkt(const crypto_session_entry_t *session, odp_packet_t pkt) +{ + const uint32_t new_headroom = CONFIG_PACKET_HEADROOM; + uint32_t headroom; + uint32_t len; + uint32_t shift; + int rc; + + if (odp_likely(odp_packet_num_segs(pkt) == 1)) + return 0; + if (session->flags.chained_bufs_ok) + return 0; + + headroom = odp_packet_headroom(pkt); + if (odp_unlikely(new_headroom >= headroom)) + return -1; + + len = odp_packet_len(pkt); + shift = headroom - new_headroom; + odp_packet_push_head(pkt, shift); + odp_packet_move_data(pkt, 0, shift, len); + /* We rely on our trunc implementation to not change the handle */ + rc = odp_packet_trunc_tail(&pkt, shift, NULL, NULL); + ODP_ASSERT(rc == 0); + + return odp_packet_num_segs(pkt) != 1; +} + static int odp_crypto_int(odp_packet_t pkt_in, odp_packet_t *pkt_out, @@ -1896,6 +1976,9 @@ int odp_crypto_int(odp_packet_t pkt_in, pkt_in = ODP_PACKET_INVALID; } + if (linearize_pkt(session, out_pkt)) + goto err; + rte_session = session->rte_session; /* NULL rte_session means that it is a NULL-NULL operation. * Just return new packet. */ @@ -1914,7 +1997,7 @@ int odp_crypto_int(odp_packet_t pkt_in, int retry_count = 0; int queue_pair; int rc; - odp_bool_t queue_pairs_shared = session->cdev_qpairs_shared; + odp_bool_t queue_pairs_shared = session->flags.cdev_qpairs_shared; if (odp_unlikely(queue_pairs_shared)) queue_pair = odp_thread_id() % session->cdev_nb_qpairs; diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c index 914a48fad..45f00f8ac 100644 --- a/platform/linux-dpdk/odp_init.c +++ b/platform/linux-dpdk/odp_init.c @@ -74,6 +74,7 @@ static void disable_features(odp_global_data_ro_t *global_ro, if (disable_ipsec && disable_crypto) global_ro->disable.crypto = 1; + global_ro->disable.stash = init_param->not_used.feat.stash; global_ro->disable.traffic_mngr = init_param->not_used.feat.tm; global_ro->disable.compress = init_param->not_used.feat.compress; } diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index f5af3ce72..065a182a8 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -9,6 +9,7 @@ #include <odp/api/plat/packet_inlines.h> #include <odp_packet_internal.h> #include <odp_debug_internal.h> +#include <odp_macros_internal.h> #include <odp_chksum_internal.h> #include <odp/api/hints.h> #include <odp/api/byteorder.h> @@ -365,8 +366,8 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, } /* Expand the original head segment*/ newhead->pkt_len += rte_pktmbuf_headroom(mb); + mb->data_len += rte_pktmbuf_headroom(mb); mb->data_off = 0; - mb->data_len = mb->buf_len; _copy_head_metadata(newhead, mb); mb = newhead; *pkt = (odp_packet_t)newhead; @@ -522,11 +523,18 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, void **tail_ptr, uint32_t *tailroom) { struct rte_mbuf *mb = pkt_to_mbuf(*pkt); + struct rte_mbuf *last_mb = rte_pktmbuf_lastseg(mb); if (odp_unlikely(len >= odp_packet_len(*pkt))) return -1; - if (rte_pktmbuf_trim(mb, len)) { + /* + * Trim only if the last segment does not become zero length. + */ + if (odp_likely(len < last_mb->data_len)) { + if (odp_unlikely(rte_pktmbuf_trim(mb, len))) + return -1; + } else { struct rte_mbuf *reverse[mb->nb_segs]; struct rte_mbuf *t = mb; int i; @@ -1311,7 +1319,15 @@ static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr, return sum; } -/** Parser helper function for Ethernet packets */ +/* + * In the worst case we look at the Ethernet header, 8 bytes of LLC/SNAP + * header and two VLAN tags in the same packet. + */ +#define PARSE_ETH_BYTES (sizeof(_odp_ethhdr_t) + 8 + 2 * sizeof(_odp_vlanhdr_t)) +/** Parser helper function for Ethernet packets + * + * Requires up to PARSE_ETH_BYTES bytes of contiguous packet data. + */ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len) { @@ -1328,7 +1344,7 @@ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, eth = (const _odp_ethhdr_t *)*parseptr; /* Detect jumbo frames */ - if (odp_unlikely(frame_len > _ODP_ETH_LEN_MAX)) + if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX)) input_flags.jumbo = 1; /* Handle Ethernet broadcast/multicast addresses */ @@ -1386,14 +1402,27 @@ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_vlanhdr_t); } + /* + * The packet was too short for what we parsed. We just give up + * entirely without trying to parse what fits in the packet. + */ + if (odp_unlikely(*offset > frame_len)) { + input_flags.all = 0; + input_flags.l2 = 1; + ethtype = 0; + } + error: prs->input_flags.all |= input_flags.all; return ethtype; } +#define PARSE_IPV4_BYTES (0xfU * 4) /* max IPv4 header length with options */ /** * Parser helper function for IPv4 + * + * Requires up to PARSE_IPV4_BYTES bytes of contiguous packet data. */ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, @@ -1409,6 +1438,7 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN || ver != 4 || + sizeof(*ipv4) > frame_len - *offset || (l3_len > frame_len - *offset))) { prs->flags.ip_err = 1; return 0; @@ -1451,8 +1481,15 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, return ipv4->proto; } +/* + * Peeks 2 bytes beyond IPv6 base header without length check if there + * are extension headers. + */ +#define PARSE_IPV6_BYTES (sizeof(_odp_ipv6hdr_t) + 2) /** * Parser helper function for IPv6 + * + * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data. */ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, @@ -1467,8 +1504,9 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, _ODP_IPV6HDR_LEN; /* Basic sanity checks on IPv6 header */ - if ((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || - l3_len > frame_len - *offset) { + if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || + sizeof(*ipv6) > frame_len - *offset || + l3_len > frame_len - *offset)) { prs->flags.ip_err = 1; return 0; } @@ -1520,8 +1558,11 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, return ipv6->next_hdr; } +#define PARSE_TCP_BYTES (sizeof(_odp_tcphdr_t)) /** * Parser helper function for TCP + * + * Requires PARSE_TCP_BYTES bytes of contiguous packet data. */ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t tcp_len, @@ -1547,8 +1588,15 @@ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += len; } +/* + * In the worst case we look at the UDP header and 4 bytes of the UDP + * payload (the non-ESP marker to distinguish IKE packets from ESP packets). + */ +#define PARSE_UDP_BYTES (sizeof(_odp_udphdr_t) + 4) /** * Parser helper function for UDP + * + * Requires PARSE_UDP_BYTES bytes of contiguous packet data. */ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, odp_proto_chksums_t chksums, @@ -1593,8 +1641,11 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_udphdr_t); } +#define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t)) /** * Parser helper function for SCTP + * + * Requires PARSE_SCTP_BYTES bytes of contiguous packet data. */ static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t sctp_len, @@ -1621,6 +1672,10 @@ static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_sctphdr_t); } +#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c))) +#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \ + MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES)) +/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */ static inline int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, uint32_t offset, @@ -1984,6 +2039,8 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, if (chksums.chksum.sctp && pkt_hdr->p.input_flags.sctp && !pkt_hdr->p.input_flags.ipfrag) { + uint32_t seg_len = 0; + _odp_sctphdr_t hdr_copy; uint32_t sum = ~packet_sum_crc32c(pkt_hdr, pkt_hdr->p.l4_offset + _ODP_SCTPHDR_LEN, @@ -1993,8 +2050,14 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, l4_part_sum); _odp_sctphdr_t *sctp = odp_packet_offset(packet_handle(pkt_hdr), pkt_hdr->p.l4_offset, - NULL, NULL); + &seg_len, NULL); + if (odp_unlikely(seg_len < sizeof(*sctp))) { + odp_packet_t pkt = packet_handle(pkt_hdr); + sctp = &hdr_copy; + odp_packet_copy_to_mem(pkt, pkt_hdr->p.l4_offset, + sizeof(*sctp), sctp); + } pkt_hdr->p.input_flags.l4_chksum_done = 1; if (sum != sctp->chksum) { pkt_hdr->p.flags.l4_chksum_err = 1; @@ -2051,12 +2114,14 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); const uint8_t *data; uint32_t seg_len; - uint32_t len = odp_packet_len(pkt); + uint32_t packet_len = odp_packet_len(pkt); odp_proto_t proto = param->proto; odp_proto_layer_t layer = param->last_layer; int ret; uint16_t ethtype; uint64_t l4_part_sum = 0; + const uint32_t min_seglen = PARSE_ETH_BYTES + PARSE_L3_L4_BYTES; + uint8_t buf[min_seglen]; if (proto == ODP_PROTO_NONE || layer == ODP_PROTO_LAYER_NONE) return -1; @@ -2066,6 +2131,20 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, if (data == NULL) return -1; + /* + * We must not have a packet segment boundary within the parsed + * packet data range. Copy enough data to a temporary buffer for + * parsing if necessary. + */ + if (odp_unlikely(pkt_hdr->buf_hdr.mb.nb_segs > 1) && + odp_unlikely(seg_len < min_seglen)) { + seg_len = min_seglen; + if (seg_len > packet_len - offset) + seg_len = packet_len - offset; + odp_packet_copy_to_mem(pkt, offset, seg_len, buf); + data = buf; + } + /* Reset parser flags, keep other flags */ packet_parse_reset(pkt_hdr, 0); @@ -2073,7 +2152,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, /* Assume valid L2 header, no CRC/FCS check in SW */ pkt_hdr->p.l2_offset = offset; - ethtype = parse_eth(&pkt_hdr->p, &data, &offset, len); + ethtype = parse_eth(&pkt_hdr->p, &data, &offset, packet_len); } else if (proto == ODP_PROTO_IPV4) { ethtype = _ODP_ETHTYPE_IPV4; } else if (proto == ODP_PROTO_IPV6) { @@ -2083,7 +2162,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, } ret = packet_parse_common_l3_l4(&pkt_hdr->p, data, offset, - len, seg_len, + packet_len, seg_len, layer, ethtype, param->chksums, &l4_part_sum); @@ -2404,6 +2483,13 @@ odp_packet_reass_status_t odp_packet_reass_status(odp_packet_t pkt) return ODP_PACKET_REASS_NONE; } +int odp_packet_reass_info(odp_packet_t pkt, odp_packet_reass_info_t *info) +{ + (void)pkt; + (void)info; + return -1; +} + int odp_packet_reass_partial_state(odp_packet_t pkt, odp_packet_t frags[], odp_packet_reass_partial_state_t *res) { diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index 4b5eb3206..4417ca4ee 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -153,7 +153,7 @@ int _odp_pool_init_global(void) uint32_t i; odp_shm_t shm; - shm = odp_shm_reserve("_odp_pool_glb", sizeof(pool_global_t), + shm = odp_shm_reserve("_odp_pool_global", sizeof(pool_global_t), ODP_CACHE_LINE_SIZE, 0); _odp_pool_glb = odp_shm_addr(shm); diff --git a/platform/linux-dpdk/odp_queue_basic.c b/platform/linux-dpdk/odp_queue_basic.c index 7145fb2ce..33e0ba6bc 100644 --- a/platform/linux-dpdk/odp_queue_basic.c +++ b/platform/linux-dpdk/odp_queue_basic.c @@ -130,7 +130,7 @@ static int queue_init_global(void) _odp_queue_inline_offset.context = offsetof(queue_entry_t, s.param.context); - shm = odp_shm_reserve("_odp_queue_gbl", + shm = odp_shm_reserve("_odp_queue_basic_global", sizeof(queue_global_t), sizeof(queue_entry_t), 0); @@ -634,7 +634,8 @@ static void queue_print(odp_queue_t handle) odp_pktio_info_t pktio_info; queue_entry_t *queue; uint32_t queue_id; - int status; + int status, prio; + int max_prio = odp_schedule_max_prio(); queue_id = queue_to_index(handle); @@ -658,7 +659,7 @@ static void queue_print(odp_queue_t handle) ODP_PRINT("\nQueue info\n"); ODP_PRINT("----------\n"); ODP_PRINT(" handle %p\n", queue->s.handle); - ODP_PRINT(" index %" PRIu32 "\n", queue->s.index); + ODP_PRINT(" index %" PRIu32 "\n", queue_id); ODP_PRINT(" name %s\n", queue->s.name); ODP_PRINT(" enq mode %s\n", queue->s.param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : @@ -686,8 +687,11 @@ static void queue_print(odp_queue_t handle) "ODP_SCHED_SYNC_ATOMIC" : (queue->s.param.sched.sync == ODP_SCHED_SYNC_ORDERED ? "ODP_SCHED_SYNC_ORDERED" : "unknown"))); - ODP_PRINT(" priority %d\n", queue->s.param.sched.prio); - ODP_PRINT(" group %d\n", queue->s.param.sched.group); + prio = queue->s.param.sched.prio; + ODP_PRINT(" priority %i (%i in API)\n", max_prio - prio, prio); + ODP_PRINT(" group %i\n", queue->s.param.sched.group); + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + ODP_PRINT(" spread %i\n", _odp_sched_basic_get_spread(queue_id)); } if (queue->s.pktin.pktio != ODP_PKTIO_INVALID) { if (!odp_pktio_info(queue->s.pktin.pktio, &pktio_info)) @@ -744,11 +748,18 @@ static void queue_print_all(void) char type_c, enq_c, deq_c, order_c, sync_c; const int col_width = 24; int prio = 0; + int spr = 0; odp_schedule_sync_t sync = ODP_SCHED_SYNC_PARALLEL; + odp_schedule_group_t grp = ODP_SCHED_GROUP_INVALID; ODP_PRINT("\nList of all queues\n"); ODP_PRINT("------------------\n"); - ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio\n", col_width, "name"); + ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio grp", + col_width, "name"); + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + ODP_PRINT(" spr\n"); + else + ODP_PRINT("\n"); for (i = 0; i < CONFIG_MAX_QUEUES; i++) { queue_entry_t *queue = qentry_from_index(i); @@ -777,7 +788,10 @@ static void queue_print_all(void) len = ring_st_length(queue->s.ring_st); max_len = ring_st_max_length(queue->s.ring_st); prio = queue->s.param.sched.prio; + grp = queue->s.param.sched.group; sync = queue->s.param.sched.sync; + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + spr = _odp_sched_basic_get_spread(index); } else { len = ring_mpmc_length(queue->s.ring_mpmc); max_len = ring_mpmc_max_length(queue->s.ring_mpmc); @@ -811,7 +825,13 @@ static void queue_print_all(void) if (type == ODP_QUEUE_TYPE_SCHED) { sync_c = (sync == ODP_SCHED_SYNC_PARALLEL) ? 'P' : ((sync == ODP_SCHED_SYNC_ATOMIC) ? 'A' : 'O'); - ODP_PRINT(" %c %4i", sync_c, prio); + /* Print prio level matching odp_schedule_print() output */ + prio = odp_schedule_max_prio() - prio; + + ODP_PRINT(" %c %4i %3i", sync_c, prio, grp); + + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + ODP_PRINT(" %3i", spr); } ODP_PRINT("\n"); diff --git a/platform/linux-dpdk/odp_queue_eventdev.c b/platform/linux-dpdk/odp_queue_eventdev.c index 96baffa6f..0960c456e 100644 --- a/platform/linux-dpdk/odp_queue_eventdev.c +++ b/platform/linux-dpdk/odp_queue_eventdev.c @@ -528,7 +528,7 @@ static int queue_init_global(void) _odp_queue_inline_offset.context = offsetof(queue_entry_t, s.param.context); - shm = odp_shm_reserve("_odp_eventdev_gbl", + shm = odp_shm_reserve("_odp_queue_eventdev_global", sizeof(eventdev_global_t), ODP_CACHE_LINE_SIZE, 0); diff --git a/platform/linux-dpdk/odp_schedule_if.c b/platform/linux-dpdk/odp_schedule_if.c index 29d38b1fd..21f5fc85b 100644 --- a/platform/linux-dpdk/odp_schedule_if.c +++ b/platform/linux-dpdk/odp_schedule_if.c @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -11,6 +12,9 @@ #include <odp_debug_internal.h> #include <odp_global_data.h> +/* Required for _ODP_SCHED_ID_EVENTDEV */ +#include <odp_eventdev_internal.h> + #include <stdlib.h> #include <string.h> @@ -25,6 +29,7 @@ extern const schedule_api_t _odp_schedule_eventdev_api; const schedule_fn_t *_odp_sched_fn; const schedule_api_t *_odp_sched_api; +int _odp_sched_id; uint64_t odp_schedule_wait_time(uint64_t ns) { @@ -216,12 +221,15 @@ int _odp_schedule_init_global(void) ODP_PRINT("Using scheduler '%s'\n", sched); if (!strcmp(sched, "basic")) { + _odp_sched_id = _ODP_SCHED_ID_BASIC; _odp_sched_fn = &_odp_schedule_basic_fn; _odp_sched_api = &_odp_schedule_basic_api; } else if (!strcmp(sched, "sp")) { + _odp_sched_id = _ODP_SCHED_ID_SP; _odp_sched_fn = &_odp_schedule_sp_fn; _odp_sched_api = &_odp_schedule_sp_api; } else if (!strcmp(sched, "eventdev")) { + _odp_sched_id = _ODP_SCHED_ID_EVENTDEV; _odp_sched_fn = &_odp_schedule_eventdev_fn; _odp_sched_api = &_odp_schedule_eventdev_api; } else { diff --git a/platform/linux-dpdk/odp_shared_memory.c b/platform/linux-dpdk/odp_shared_memory.c index 4b1432a2f..645bb8847 100644 --- a/platform/linux-dpdk/odp_shared_memory.c +++ b/platform/linux-dpdk/odp_shared_memory.c @@ -1,4 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -25,9 +26,7 @@ #define SHM_MAX_ALIGN (0x80000000) #define SHM_BLOCK_NAME "%" PRIu64 "-%d-%s" - -ODP_STATIC_ASSERT(ODP_CONFIG_SHM_BLOCKS >= ODP_CONFIG_POOLS, - "ODP_CONFIG_SHM_BLOCKS < ODP_CONFIG_POOLS"); +#define SHM_MAX_NB_BLOCKS (CONFIG_INTERNAL_SHM_BLOCKS + CONFIG_SHM_BLOCKS) ODP_STATIC_ASSERT(ODP_SHM_NAME_LEN >= RTE_MEMZONE_NAMESIZE, "ODP_SHM_NAME_LEN < RTE_MEMZONE_NAMESIZE"); @@ -68,7 +67,7 @@ typedef struct { */ typedef struct { odp_spinlock_t lock; - shm_block_t block[ODP_CONFIG_SHM_BLOCKS]; + shm_block_t block[SHM_MAX_NB_BLOCKS]; } shm_table_t; static shm_table_t *shm_tbl; @@ -80,7 +79,7 @@ static odp_bool_t mz_name_used(const char *name) { int idx; - for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) { + for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) { if (shm_tbl->block[idx].mz && strncmp(name, shm_tbl->block[idx].mz->name, RTE_MEMZONE_NAMESIZE) == 0) @@ -124,7 +123,7 @@ static int find_free_block(void) { int idx; - for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) { + for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) { if (shm_tbl->block[idx].mz == NULL) return idx; } @@ -145,7 +144,7 @@ static inline odp_bool_t handle_is_valid(odp_shm_t shm) { int idx = handle_to_idx(shm); - if (idx < 0 || idx >= ODP_CONFIG_SHM_BLOCKS || + if (idx < 0 || idx >= SHM_MAX_NB_BLOCKS || shm_tbl->block[idx].mz == NULL) { ODP_ERR("Invalid odp_shm_t handle: %" PRIu64 "\n", odp_shm_to_u64(shm)); @@ -199,7 +198,7 @@ int _odp_shm_term_global(void) } /* Cleanup possibly non freed memory (and complain a bit) */ - for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) { + for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) { block = &shm_tbl->block[idx]; if (block->mz) { ODP_ERR("block '%s' was never freed (cleaning up...)\n", @@ -224,7 +223,7 @@ int odp_shm_capability(odp_shm_capability_t *capa) { memset(capa, 0, sizeof(odp_shm_capability_t)); - capa->max_blocks = ODP_CONFIG_SHM_BLOCKS; + capa->max_blocks = CONFIG_SHM_BLOCKS; capa->max_size = 0; capa->max_align = SHM_MAX_ALIGN; @@ -357,7 +356,7 @@ odp_shm_t odp_shm_lookup(const char *name) odp_spinlock_lock(&shm_tbl->lock); - for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) { + for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) { if (shm_tbl->block[idx].mz && strncmp(name, shm_tbl->block[idx].name, ODP_SHM_NAME_LEN) == 0) { @@ -425,7 +424,7 @@ void odp_shm_print_all(void) ODP_PRINT("\nShared memory blocks\n--------------------\n"); - for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) { + for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) { block = &shm_tbl->block[idx]; if (block->mz == NULL) continue; diff --git a/platform/linux-dpdk/odp_system_info.c b/platform/linux-dpdk/odp_system_info.c index af07f387a..30fcc140b 100644 --- a/platform/linux-dpdk/odp_system_info.c +++ b/platform/linux-dpdk/odp_system_info.c @@ -515,7 +515,7 @@ void odp_sys_config_print(void) ODP_PRINT("CONFIG_PACKET_SEG_SIZE: %i\n", CONFIG_PACKET_SEG_SIZE); ODP_PRINT("CONFIG_PACKET_SEG_LEN_MIN: %i\n", CONFIG_PACKET_SEG_LEN_MIN); ODP_PRINT("CONFIG_PACKET_MAX_SEG_LEN: %i\n", CONFIG_PACKET_MAX_SEG_LEN); - ODP_PRINT("ODP_CONFIG_SHM_BLOCKS: %i\n", ODP_CONFIG_SHM_BLOCKS); + ODP_PRINT("CONFIG_SHM_BLOCKS: %i\n", CONFIG_SHM_BLOCKS); ODP_PRINT("CONFIG_BURST_SIZE: %i\n", CONFIG_BURST_SIZE); ODP_PRINT("CONFIG_POOL_MAX_NUM: %i\n", CONFIG_POOL_MAX_NUM); ODP_PRINT("\n"); diff --git a/platform/linux-dpdk/odp_thread.c b/platform/linux-dpdk/odp_thread.c index 59394b3e4..7ab41cf72 100644 --- a/platform/linux-dpdk/odp_thread.c +++ b/platform/linux-dpdk/odp_thread.c @@ -76,7 +76,7 @@ int _odp_thread_init_global(void) if (num_max > ODP_THREAD_COUNT_MAX) num_max = ODP_THREAD_COUNT_MAX; - shm = odp_shm_reserve("_odp_thread_globals", + shm = odp_shm_reserve("_odp_thread_global", sizeof(thread_globals_t), ODP_CACHE_LINE_SIZE, 0); diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 8c75e5ec0..e763c0abc 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -275,12 +275,13 @@ __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/arm/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/arm/odp/api/abi/cpu_inlines.h \ arch/arm/odp/api/abi/cpu.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ @@ -297,14 +298,15 @@ __LIB__libodp_linux_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/aarch64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/aarch64/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu_time.h \ arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/aarch64/odp/api/abi/atomic_inlines.h \ arch/aarch64/odp/api/abi/atomic.h \ - arch/aarch64/odp/api/abi/cpu.h + arch/default/odp/api/abi/cpu_generic.h \ + arch/aarch64/odp/api/abi/cpu_inlines.h \ + arch/aarch64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ @@ -317,12 +319,13 @@ __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -331,16 +334,16 @@ noinst_HEADERS += arch/default/odp_atomic.h \ endif if ARCH_IS_MIPS64 __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ - arch/mips64/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/mips64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/mips64/odp/api/abi/cpu_inlines.h \ arch/mips64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -353,12 +356,13 @@ __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/powerpc/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/default/odp/api/abi/cpu_inlines.h \ arch/powerpc/odp/api/abi/cpu.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -372,13 +376,13 @@ __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/x86/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_inlines.h \ - arch/x86/odp/api/abi/cpu_rdtsc.h \ +odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_rdtsc.h \ arch/x86/odp/api/abi/cpu_time.h \ arch/x86/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/x86/odp/api/abi/cpu_inlines.h \ arch/x86/odp/api/abi/cpu.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h index d1dbf36b8..14cca3ca0 100644 --- a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h @@ -9,4 +9,4 @@ #endif #include <odp/api/abi-default/atomic.h> - +#include <odp/api/plat/atomic_inlines.h> diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu.h b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu.h index 97a2861c5..825ff19d4 100644 --- a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu.h +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu.h @@ -17,15 +17,8 @@ extern "C" { #define ODP_CACHE_LINE_SIZE _ODP_CACHE_LINE_SIZE #endif -static inline void odp_cpu_pause(void) -{ - /* YIELD hints the CPU to switch to another thread if possible - * and executes as a NOP otherwise. - * ISB flushes the pipeline, then restarts. This is guaranteed to - * stall the CPU a number of cycles. - */ - __asm volatile("isb" ::: "memory"); -} +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/cpu_inlines.h> #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_inlines.h new file mode 100644 index 000000000..bf44806a0 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_inlines.h @@ -0,0 +1,32 @@ +/* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_CPU_INLINES_H_ +#define ODP_ARCH_CPU_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_cpu_pause(void) +{ + /* YIELD hints the CPU to switch to another thread if possible + * and executes as a NOP otherwise. + * ISB flushes the pipeline, then restarts. This is guaranteed to + * stall the CPU a number of cycles. + */ + __asm volatile("isb" ::: "memory"); +} + +/* Use generic implementations for the rest of the functions */ +#include <odp/api/abi/cpu_generic.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/arm/odp/api/abi/cpu.h b/platform/linux-generic/arch/arm/odp/api/abi/cpu.h index 0ab5b8e14..9224af9a0 100644 --- a/platform/linux-generic/arch/arm/odp/api/abi/cpu.h +++ b/platform/linux-generic/arch/arm/odp/api/abi/cpu.h @@ -13,15 +13,8 @@ extern "C" { #define ODP_CACHE_LINE_SIZE 64 -static inline void odp_cpu_pause(void) -{ - /* YIELD hints the CPU to switch to another thread if possible - * and executes as a NOP otherwise. - * ISB flushes the pipeline, then restarts. This is guaranteed to - * stall the CPU a number of cycles. - */ - __asm volatile("isb" ::: "memory"); -} +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/cpu_inlines.h> #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/arm/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/arm/odp/api/abi/cpu_inlines.h new file mode 100644 index 000000000..bf44806a0 --- /dev/null +++ b/platform/linux-generic/arch/arm/odp/api/abi/cpu_inlines.h @@ -0,0 +1,32 @@ +/* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_CPU_INLINES_H_ +#define ODP_ARCH_CPU_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_cpu_pause(void) +{ + /* YIELD hints the CPU to switch to another thread if possible + * and executes as a NOP otherwise. + * ISB flushes the pipeline, then restarts. This is guaranteed to + * stall the CPU a number of cycles. + */ + __asm volatile("isb" ::: "memory"); +} + +/* Use generic implementations for the rest of the functions */ +#include <odp/api/abi/cpu_generic.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu.h b/platform/linux-generic/arch/default/odp/api/abi/cpu.h index 8f64790c3..e09efdfcf 100644 --- a/platform/linux-generic/arch/default/odp/api/abi/cpu.h +++ b/platform/linux-generic/arch/default/odp/api/abi/cpu.h @@ -13,9 +13,8 @@ extern "C" { #define ODP_CACHE_LINE_SIZE 64 -static inline void odp_cpu_pause(void) -{ -} +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/cpu_inlines.h> #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu_generic.h b/platform/linux-generic/arch/default/odp/api/abi/cpu_generic.h new file mode 100644 index 000000000..b75e65717 --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/cpu_generic.h @@ -0,0 +1,34 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_CPU_GENERIC_H_ +#define ODP_API_ABI_CPU_GENERIC_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +uint64_t _odp_cpu_cycles(void); +int _odp_cpu_cycles_init_global(void); + +static inline uint64_t _odp_cpu_cycles_max(void) +{ + return UINT64_MAX; +} + +static inline uint64_t _odp_cpu_cycles_resolution(void) +{ + return 1; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/cpu_inlines.h index f9c2bae9d..54aeae946 100644 --- a/platform/linux-generic/arch/default/odp/api/abi/cpu_inlines.h +++ b/platform/linux-generic/arch/default/odp/api/abi/cpu_inlines.h @@ -7,9 +7,18 @@ #ifndef ODP_ARCH_CPU_INLINES_H_ #define ODP_ARCH_CPU_INLINES_H_ -#undef odp_cpu_pause -#undef odp_cpu_cycles -#undef odp_cpu_cycles_max -#undef odp_cpu_cycles_resolution +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_cpu_pause(void) +{ +} + +#include <odp/api/abi/cpu_generic.h> + +#ifdef __cplusplus +} +#endif #endif diff --git a/platform/linux-generic/arch/default/odp_cpu_cycles.c b/platform/linux-generic/arch/default/odp_cpu_cycles.c index 542a68dbe..5d0d5db1d 100644 --- a/platform/linux-generic/arch/default/odp_cpu_cycles.c +++ b/platform/linux-generic/arch/default/odp_cpu_cycles.c @@ -1,4 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -6,16 +7,19 @@ #include <odp_posix_extensions.h> +#include <stdint.h> #include <stdlib.h> #include <time.h> -#include <odp/api/cpu.h> #include <odp_debug_internal.h> +#include <odp_global_data.h> #include <odp_init_internal.h> #define GIGA 1000000000 -uint64_t odp_cpu_cycles(void) +#include <odp/api/abi/cpu_generic.h> + +uint64_t _odp_cpu_cycles(void) { struct timespec time; uint64_t sec, ns, hz, cycles; @@ -26,7 +30,8 @@ uint64_t odp_cpu_cycles(void) if (ret != 0) ODP_ABORT("clock_gettime failed\n"); - hz = odp_cpu_hz_max(); + hz = odp_global_ro.system_info.cpu_hz_max[0]; + sec = (uint64_t)time.tv_sec; ns = (uint64_t)time.tv_nsec; @@ -36,16 +41,6 @@ uint64_t odp_cpu_cycles(void) return cycles; } -uint64_t odp_cpu_cycles_max(void) -{ - return UINT64_MAX; -} - -uint64_t odp_cpu_cycles_resolution(void) -{ - return 1; -} - int _odp_cpu_cycles_init_global(void) { return 0; diff --git a/platform/linux-generic/arch/mips64/odp/api/abi/cpu.h b/platform/linux-generic/arch/mips64/odp/api/abi/cpu.h index 45d95e9e3..a6ce523d0 100644 --- a/platform/linux-generic/arch/mips64/odp/api/abi/cpu.h +++ b/platform/linux-generic/arch/mips64/odp/api/abi/cpu.h @@ -17,13 +17,8 @@ extern "C" { #error Please add support for your arch in cpu_arch.h #endif -static inline void odp_cpu_pause(void) -{ - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); -} +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/cpu_inlines.h> #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/mips64/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/mips64/odp/api/abi/cpu_inlines.h new file mode 100644 index 000000000..d3a424432 --- /dev/null +++ b/platform/linux-generic/arch/mips64/odp/api/abi/cpu_inlines.h @@ -0,0 +1,56 @@ +/* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_CPU_INLINES_H_ +#define ODP_ARCH_CPU_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +static inline void _odp_cpu_pause(void) +{ + __asm__ __volatile__ ("nop"); + __asm__ __volatile__ ("nop"); + __asm__ __volatile__ ("nop"); + __asm__ __volatile__ ("nop"); +} + +uint64_t _odp_cpu_cycles(void) +{ + #define CVMX_TMP_STR(x) CVMX_TMP_STR2(x) + #define CVMX_TMP_STR2(x) #x + uint64_t cycle; + + __asm__ __volatile__ ("rdhwr %[rt],$" CVMX_TMP_STR(31) : + [rt] "=d" (cycle) : : "memory"); + + return cycle; +} + +uint64_t _odp_cpu_cycles_max(void) +{ + return UINT64_MAX; +} + +uint64_t _odp_cpu_cycles_resolution(void) +{ + return 1; +} + +int _odp_cpu_cycles_init_global(void) +{ + return 0; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/mips64/odp_cpu_cycles.c b/platform/linux-generic/arch/mips64/odp_cpu_cycles.c deleted file mode 100644 index 6423af76d..000000000 --- a/platform/linux-generic/arch/mips64/odp_cpu_cycles.c +++ /dev/null @@ -1,37 +0,0 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp/api/cpu.h> -#include <odp/api/hints.h> -#include <odp/api/system_info.h> -#include <odp_init_internal.h> - -uint64_t odp_cpu_cycles(void) -{ - #define CVMX_TMP_STR(x) CVMX_TMP_STR2(x) - #define CVMX_TMP_STR2(x) #x - uint64_t cycle; - - __asm__ __volatile__ ("rdhwr %[rt],$" CVMX_TMP_STR(31) : - [rt] "=d" (cycle) : : "memory"); - - return cycle; -} - -uint64_t odp_cpu_cycles_max(void) -{ - return UINT64_MAX; -} - -uint64_t odp_cpu_cycles_resolution(void) -{ - return 1; -} - -int _odp_cpu_cycles_init_global(void) -{ - return 0; -} diff --git a/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h b/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h index 9e3338d60..ecf56e82e 100644 --- a/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h +++ b/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h @@ -1,9 +1,25 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#define _ODP_NEED_GENERIC_CPU_PAUSE +#ifndef ODP_API_ABI_CPU_H_ +#define ODP_API_ABI_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + #define ODP_CACHE_LINE_SIZE 128 -#include <odp/api/abi-default/cpu.h> + +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/cpu_inlines.h> + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/platform/linux-generic/arch/x86/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/x86/odp/api/abi/cpu_inlines.h index 1eb8a9561..4b542a577 100644 --- a/platform/linux-generic/arch/x86/odp/api/abi/cpu_inlines.h +++ b/platform/linux-generic/arch/x86/odp/api/abi/cpu_inlines.h @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,10 +8,14 @@ #ifndef ODP_ARCH_CPU_INLINES_H_ #define ODP_ARCH_CPU_INLINES_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include <stdint.h> #include <odp/api/abi/cpu_rdtsc.h> -_ODP_INLINE void odp_cpu_pause(void) +static inline void _odp_cpu_pause(void) { #ifdef __SSE2__ __asm__ __volatile__ ("pause"); @@ -19,19 +24,23 @@ _ODP_INLINE void odp_cpu_pause(void) #endif } -_ODP_INLINE uint64_t odp_cpu_cycles(void) +static inline uint64_t _odp_cpu_cycles(void) { return _odp_cpu_rdtsc(); } -_ODP_INLINE uint64_t odp_cpu_cycles_max(void) +static inline uint64_t _odp_cpu_cycles_max(void) { return UINT64_MAX; } -_ODP_INLINE uint64_t odp_cpu_cycles_resolution(void) +static inline uint64_t _odp_cpu_cycles_resolution(void) { return 1; } +#ifdef __cplusplus +} +#endif + #endif diff --git a/platform/linux-generic/include-abi/odp/api/abi/atomic.h b/platform/linux-generic/include-abi/odp/api/abi/atomic.h index 7c11b0ab2..9c87f9cb8 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/atomic.h +++ b/platform/linux-generic/include-abi/odp/api/abi/atomic.h @@ -84,7 +84,6 @@ typedef struct ODP_ALIGNED(sizeof(odp_u128_t)) odp_atomic_u128_s { * @{ */ -#define _ODP_INLINE static inline #include <odp/api/plat/atomic_inlines.h> /** diff --git a/platform/linux-generic/include-abi/odp/api/abi/std.h b/platform/linux-generic/include-abi/odp/api/abi/std.h index 175b606c5..201fca18e 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/std.h +++ b/platform/linux-generic/include-abi/odp/api/abi/std.h @@ -17,7 +17,6 @@ extern "C" { #endif -#define _ODP_INLINE static inline #include <odp/api/plat/std_inlines.h> #ifdef __cplusplus diff --git a/platform/linux-generic/include-abi/odp/api/abi/sync.h b/platform/linux-generic/include-abi/odp/api/abi/sync.h index cbb6f753e..236e92c8c 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/sync.h +++ b/platform/linux-generic/include-abi/odp/api/abi/sync.h @@ -21,7 +21,6 @@ extern "C" { * @{ */ -#define _ODP_INLINE static inline #include <odp/api/plat/sync_inlines.h> /** diff --git a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h index 4ab8bb411..a9da70890 100644 --- a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h @@ -34,6 +34,13 @@ #define odp_atomic_dec_u32 __odp_atomic_dec_u32 #define odp_atomic_cas_u32 __odp_atomic_cas_u32 #define odp_atomic_xchg_u32 __odp_atomic_xchg_u32 + #define odp_atomic_load_acq_u32 __odp_atomic_load_acq_u32 + #define odp_atomic_store_rel_u32 __odp_atomic_store_rel_u32 + #define odp_atomic_add_rel_u32 __odp_atomic_add_rel_u32 + #define odp_atomic_sub_rel_u32 __odp_atomic_sub_rel_u32 + #define odp_atomic_cas_acq_u32 __odp_atomic_cas_acq_u32 + #define odp_atomic_cas_rel_u32 __odp_atomic_cas_rel_u32 + #define odp_atomic_cas_acq_rel_u32 __odp_atomic_cas_acq_rel_u32 #define odp_atomic_max_u32 __odp_atomic_max_u32 #define odp_atomic_min_u32 __odp_atomic_min_u32 #define odp_atomic_init_u64 __odp_atomic_init_u64 @@ -56,35 +63,8 @@ #define odp_atomic_cas_acq_u64 __odp_atomic_cas_acq_u64 #define odp_atomic_cas_rel_u64 __odp_atomic_cas_rel_u64 #define odp_atomic_cas_acq_rel_u64 __odp_atomic_cas_acq_rel_u64 - #define odp_atomic_init_u64 __odp_atomic_init_u64 - #define odp_atomic_load_u64 __odp_atomic_load_u64 - #define odp_atomic_store_u64 __odp_atomic_store_u64 - #define odp_atomic_fetch_add_u64 __odp_atomic_fetch_add_u64 - #define odp_atomic_add_u64 __odp_atomic_add_u64 - #define odp_atomic_fetch_sub_u64 __odp_atomic_fetch_sub_u64 - #define odp_atomic_sub_u64 __odp_atomic_sub_u64 - #define odp_atomic_fetch_inc_u64 __odp_atomic_fetch_inc_u64 - #define odp_atomic_inc_u64 __odp_atomic_inc_u64 - #define odp_atomic_fetch_dec_u64 __odp_atomic_fetch_dec_u64 - #define odp_atomic_dec_u64 __odp_atomic_dec_u64 - #define odp_atomic_cas_u64 __odp_atomic_cas_u64 - #define odp_atomic_xchg_u64 __odp_atomic_xchg_u64 - #define odp_atomic_load_acq_u64 __odp_atomic_load_acq_u64 - #define odp_atomic_store_rel_u64 __odp_atomic_store_rel_u64 - #define odp_atomic_add_rel_u64 __odp_atomic_add_rel_u64 - #define odp_atomic_sub_rel_u64 __odp_atomic_sub_rel_u64 - #define odp_atomic_cas_acq_u64 __odp_atomic_cas_acq_u64 - #define odp_atomic_cas_rel_u64 __odp_atomic_cas_rel_u64 - #define odp_atomic_cas_acq_rel_u64 __odp_atomic_cas_acq_rel_u64 #define odp_atomic_max_u64 __odp_atomic_max_u64 #define odp_atomic_min_u64 __odp_atomic_min_u64 - #define odp_atomic_load_acq_u32 __odp_atomic_load_acq_u32 - #define odp_atomic_store_rel_u32 __odp_atomic_store_rel_u32 - #define odp_atomic_add_rel_u32 __odp_atomic_add_rel_u32 - #define odp_atomic_sub_rel_u32 __odp_atomic_sub_rel_u32 - #define odp_atomic_cas_acq_u32 __odp_atomic_cas_acq_u32 - #define odp_atomic_cas_rel_u32 __odp_atomic_cas_rel_u32 - #define odp_atomic_cas_acq_rel_u32 __odp_atomic_cas_acq_rel_u32 #define odp_atomic_init_u128 __odp_atomic_init_u128 #define odp_atomic_load_u128 __odp_atomic_load_u128 #define odp_atomic_store_u128 __odp_atomic_store_u128 diff --git a/platform/linux-generic/include/odp/api/plat/cpu_inlines.h b/platform/linux-generic/include/odp/api/plat/cpu_inlines.h index 054801e80..60c4bb920 100644 --- a/platform/linux-generic/include/odp/api/plat/cpu_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/cpu_inlines.h @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -9,6 +10,10 @@ #include <odp/api/hints.h> +#include <odp/api/abi/cpu_inlines.h> + +#include <stdint.h> + /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ #ifndef _ODP_NO_INLINE @@ -23,7 +28,25 @@ #define _ODP_INLINE #endif -#include <odp/api/abi/cpu_inlines.h> +_ODP_INLINE void odp_cpu_pause(void) +{ + _odp_cpu_pause(); +} + +_ODP_INLINE uint64_t odp_cpu_cycles_max(void) +{ + return _odp_cpu_cycles_max(); +} + +_ODP_INLINE uint64_t odp_cpu_cycles_resolution(void) +{ + return _odp_cpu_cycles_resolution(); +} + +_ODP_INLINE uint64_t odp_cpu_cycles(void) +{ + return _odp_cpu_cycles(); +} _ODP_INLINE uint64_t odp_cpu_cycles_diff(uint64_t c2, uint64_t c1) { diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index d2e3c808f..e7bc78d6e 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -30,7 +30,6 @@ extern "C" { #include <odp/api/thread.h> #include <odp/api/event.h> #include <odp_forward_typedefs_internal.h> -#include <odp_schedule_if.h> #include <stddef.h> typedef union buffer_index_t { diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 1e8b390dd..40d2639f1 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -20,9 +20,9 @@ extern "C" { #define CONFIG_NUM_CPU_IDS 256 /* - * Maximum number of pools. Limited by ISHM_MAX_NB_BLOCKS. + * Maximum number of pools. */ -#define ODP_CONFIG_POOLS 55 +#define ODP_CONFIG_POOLS 32 /* * Queues reserved for ODP internal use @@ -116,11 +116,21 @@ extern "C" { CONFIG_PACKET_HEADROOM + \ CONFIG_PACKET_TAILROOM) -/* Maximum number of shared memory blocks. +/* + * Number of shared memory blocks reserved for implementation internal use. + * + * Each pool requires three SHM blocks (buffers, ring, user area). 20 blocks are + * reserved for per ODP module global data. + */ +#define CONFIG_INTERNAL_SHM_BLOCKS ((ODP_CONFIG_POOLS * 3) + 20) + +/* + * Maximum number of shared memory blocks. * - * This the the number of separate SHM areas that can be reserved concurrently + * This is the number of separate SHM blocks that an application can reserve + * concurrently. */ -#define ODP_CONFIG_SHM_BLOCKS (ODP_CONFIG_POOLS + 48) +#define CONFIG_SHM_BLOCKS 64 /* * Maximum event burst size diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index a907f87be..e4fd583a4 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -74,6 +74,7 @@ typedef struct odp_global_data_ro_t { uint8_t compress; uint8_t crypto; uint8_t ipsec; + uint8_t stash; uint8_t traffic_mngr; } disable; diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index dc4754710..dc9f0d207 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -20,6 +20,7 @@ extern "C" { #include <odp/api/shared_memory.h> #include <odp/api/ticketlock.h> +#include <odp/api/align.h> #include <odp_buffer_internal.h> #include <odp_config_internal.h> @@ -94,7 +95,7 @@ typedef struct pool_t { pool_destroy_cb_fn ext_destroy; void *ext_desc; - struct ODP_CACHE_ALIGNED { + struct ODP_ALIGNED_CACHE { odp_atomic_u64_t alloc_ops; odp_atomic_u64_t alloc_fails; odp_atomic_u64_t free_ops; diff --git a/platform/linux-generic/include/odp_queue_basic_internal.h b/platform/linux-generic/include/odp_queue_basic_internal.h index 25e35b22c..60817fc75 100644 --- a/platform/linux-generic/include/odp_queue_basic_internal.h +++ b/platform/linux-generic/include/odp_queue_basic_internal.h @@ -119,6 +119,9 @@ int _odp_sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int num, int update_status); int _odp_sched_queue_empty(uint32_t queue_index); +/* Functions by schedulers */ +int _odp_sched_basic_get_spread(uint32_t queue_index); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index d3202543d..a804f8c95 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -16,8 +17,12 @@ extern "C" { #include <odp/api/schedule.h> #include <odp_forward_typedefs_internal.h> -/* Number of ordered locks per queue */ -#define SCHEDULE_ORDERED_LOCKS_PER_QUEUE 2 +#define _ODP_SCHED_ID_BASIC 0 +#define _ODP_SCHED_ID_SP 1 +#define _ODP_SCHED_ID_SCALABLE 2 + +/* Scheduler identifier */ +extern int _odp_sched_id; typedef struct schedule_config_t { struct { diff --git a/platform/linux-generic/odp_comp.c b/platform/linux-generic/odp_comp.c index 685c9098a..579bdc03b 100644 --- a/platform/linux-generic/odp_comp.c +++ b/platform/linux-generic/odp_comp.c @@ -620,7 +620,7 @@ int _odp_comp_init_global(void) mem_size = sizeof(*global); /* Allocate our globally shared memory */ - shm = odp_shm_reserve("_odp_comp_pool", mem_size, + shm = odp_shm_reserve("_odp_comp_global", mem_size, ODP_CACHE_LINE_SIZE, 0); global = odp_shm_addr(shm); diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index cf9abc99f..aa8e8a2b3 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -1,4 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -13,6 +14,7 @@ #include <odp/api/align.h> #include <odp/api/shared_memory.h> #include <odp_debug_internal.h> +#include <odp_global_data.h> #include <odp/api/hints.h> #include <odp/api/random.h> #include <odp/api/plat/packet_inlines.h> @@ -113,6 +115,11 @@ void free_session(odp_crypto_generic_session_t *session) int odp_crypto_capability(odp_crypto_capability_t *capa) { + if (odp_global_ro.disable.crypto) { + ODP_ERR("Crypto is disabled\n"); + return -1; + } + if (NULL == capa) return -1; @@ -190,6 +197,15 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, int rc; odp_crypto_generic_session_t *session; + if (odp_global_ro.disable.crypto) { + ODP_ERR("Crypto is disabled\n"); + /* Dummy output to avoid compiler warning about uninitialized + * variables */ + *status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + /* Allocate memory for this session */ session = alloc_session(); if (NULL == session) { @@ -276,12 +292,18 @@ odp_crypto_operation(odp_crypto_op_param_t *param, packet_param.auth_range = param->auth_range; rc = odp_crypto_op(¶m->pkt, &out_pkt, &packet_param, 1); - if (rc < 0) - return rc; + if (rc <= 0) + return -1; rc = odp_crypto_result(&packet_result, out_pkt); - if (rc < 0) - return rc; + if (rc < 0) { + /* + * We cannot fail since odp_crypto_op() has already processed + * the packet. Let's indicate error in the result instead. + */ + packet_hdr(out_pkt)->p.flags.crypto_err = 1; + packet_result.ok = false; + } /* Indicate to caller operation was sync */ *posted = 0; @@ -312,11 +334,16 @@ _odp_crypto_init_global(void) odp_shm_t shm; int idx; + if (odp_global_ro.disable.crypto) { + ODP_PRINT("\nODP crypto is DISABLED\n"); + return 0; + } + /* Calculate the memory size we need */ mem_size = sizeof(odp_crypto_global_t); /* Allocate our globally shared memory */ - shm = odp_shm_reserve("_odp_crypto_pool_null", mem_size, + shm = odp_shm_reserve("_odp_crypto_null_global", mem_size, ODP_CACHE_LINE_SIZE, 0); if (ODP_SHM_INVALID == shm) { @@ -346,6 +373,9 @@ int _odp_crypto_term_global(void) int count = 0; odp_crypto_generic_session_t *session; + if (odp_global_ro.disable.crypto) + return 0; + for (session = global->free; session != NULL; session = session->next) count++; if (count != MAX_SESSIONS) { @@ -353,7 +383,7 @@ int _odp_crypto_term_global(void) rc = -1; } - ret = odp_shm_free(odp_shm_lookup("_odp_crypto_pool_null")); + ret = odp_shm_free(odp_shm_lookup("_odp_crypto_null_global")); if (ret < 0) { ODP_ERR("shm free failed for _odp_crypto_pool_null\n"); rc = -1; diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 07a91cc46..d8276b4b4 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -1,4 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -438,8 +439,12 @@ void packet_aes_xcbc_mac(odp_packet_t pkt, len -= datalen; if (eoff != 0) { if (eoff + datalen > AES_BLOCK_SIZE) { - memxor(e + eoff, data, AES_BLOCK_SIZE - eoff); - datalen -= (AES_BLOCK_SIZE - eoff); + /* bytes needed to fill the partial block */ + uint32_t remaining_len = AES_BLOCK_SIZE - eoff; + + memxor(e + eoff, data, remaining_len); + datalen -= remaining_len; + data += remaining_len; eoff = 0; EVP_EncryptUpdate(ctx, e, &dummy_len, e, sizeof(e)); @@ -695,7 +700,7 @@ int packet_cmac_eia2(odp_packet_t pkt, { CMAC_CTX *ctx = local.cmac_ctx[session->idx]; void *iv_ptr; - uint32_t offset = param->auth_range.offset; + uint32_t offset = param->auth_range.offset / 8; uint32_t len = (param->auth_range.length + 7) / 8; size_t outlen; @@ -894,120 +899,146 @@ int internal_aad(EVP_CIPHER_CTX *ctx, return ret; } -static -int internal_encrypt(EVP_CIPHER_CTX *ctx, - odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param) -{ - unsigned in_pos = param->cipher_range.offset; - unsigned out_pos = param->cipher_range.offset; - unsigned in_len = param->cipher_range.length; - uint8_t block[2 * EVP_MAX_BLOCK_LENGTH]; - unsigned block_len = EVP_CIPHER_block_size(EVP_CIPHER_CTX_cipher(ctx)); - int cipher_len; - int ret; - - ODP_ASSERT(in_pos + in_len <= odp_packet_len(pkt)); - - while (in_len > 0) { - uint32_t seglen = 0; /* GCC */ - uint8_t *insegaddr = odp_packet_offset(pkt, in_pos, - &seglen, NULL); - unsigned inseglen = in_len < seglen ? in_len : seglen; - - /* There should be at least 1 additional block in out buffer */ - if (inseglen > block_len) { - unsigned part = inseglen - block_len; - - EVP_EncryptUpdate(ctx, insegaddr, &cipher_len, - insegaddr, part); - in_pos += part; - in_len -= part; - insegaddr += part; - inseglen -= part; - - out_pos += cipher_len; - } - - /* Use temporal storage */ - if (inseglen > 0) { - unsigned part = inseglen; - - EVP_EncryptUpdate(ctx, block, &cipher_len, - insegaddr, part); - in_pos += part; - in_len -= part; - insegaddr += part; - inseglen -= part; - - odp_packet_copy_from_mem(pkt, out_pos, - cipher_len, block); - out_pos += cipher_len; - } - } - - ret = EVP_EncryptFinal_ex(ctx, block, &cipher_len); - odp_packet_copy_from_mem(pkt, out_pos, cipher_len, block); +typedef int (*evp_update_t)(EVP_CIPHER_CTX *, unsigned char *, + int *, const unsigned char *, int); - return ret; -} +typedef int (*evp_final_t)(EVP_CIPHER_CTX *, unsigned char *, int *); -static -int internal_decrypt(EVP_CIPHER_CTX *ctx, - odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param) -{ - unsigned in_pos = param->cipher_range.offset; - unsigned out_pos = param->cipher_range.offset; - unsigned in_len = param->cipher_range.length; - uint8_t block[2 * EVP_MAX_BLOCK_LENGTH]; - unsigned block_len = EVP_CIPHER_block_size(EVP_CIPHER_CTX_cipher(ctx)); - int cipher_len; - int ret; +static inline int internal_crypt(EVP_CIPHER_CTX *ctx, + odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + evp_update_t EVP_update, + evp_final_t EVP_final) +{ + uint32_t in_pos = param->cipher_range.offset; + uint32_t out_pos = in_pos; + uint32_t in_len = param->cipher_range.length; + uint8_t block[EVP_MAX_BLOCK_LENGTH]; + uint32_t buffered = 0; + uint32_t block_len = EVP_CIPHER_block_size(EVP_CIPHER_CTX_cipher(ctx)); + int out_len; + int rc; ODP_ASSERT(in_pos + in_len <= odp_packet_len(pkt)); + /* + * In the following loop we process one packet segment per iteration. + * We rely on the following properties of the encrypt/decrypt update + * function with the algorithms that we use: + * + * - The function processes (and writes to output) only whole blocks. + * - Input data beyond the last full block is buffered inside OpenSSL. + * - The amount of buffered data is always less than one block. + * - Total amount of output data does not exceed the total amount + * of input data at any point. + */ while (in_len > 0) { - uint32_t seglen = 0; /* GCC */ - uint8_t *insegaddr = odp_packet_offset(pkt, in_pos, - &seglen, NULL); - unsigned inseglen = in_len < seglen ? in_len : seglen; - - /* There should be at least 1 additional block in out buffer */ - if (inseglen > block_len) { - unsigned part = inseglen - block_len; - - EVP_DecryptUpdate(ctx, insegaddr, &cipher_len, - insegaddr, part); - in_pos += part; - in_len -= part; - insegaddr += part; - inseglen -= part; - - out_pos += cipher_len; + uint32_t seglen = 0; + uint8_t *in_addr = odp_packet_offset(pkt, in_pos, + &seglen, NULL); + uint32_t len = in_len < seglen ? in_len : seglen; + + if (odp_unlikely(buffered > 0)) { + /* + * Leftover data from the previous segment is + * in the buffer inside OpenSSL. + */ + uint32_t remaining_len = block_len - buffered; + + if (odp_likely(len >= remaining_len)) { + /* + * Let's fill the buffered input data to a + * full block and get the output block to + * a memory buffer. The buffer is then copied + * to the packet, crossing segment boundary. + */ + rc = EVP_update(ctx, block, &out_len, + in_addr, remaining_len); + if (odp_unlikely(rc != 1)) + goto err; + if (odp_unlikely(out_len != (int)block_len)) + goto err; + in_addr += remaining_len; + in_pos += remaining_len; + len -= remaining_len; + in_len -= remaining_len; + buffered = 0; + rc = odp_packet_copy_from_mem(pkt, out_pos, + block_len, block); + if (odp_unlikely(rc)) + goto err; + out_pos += block_len; + } else { + /* + * Not enough data in this segment to fill + * the buffer to a full block. Fill the buffer + * a bit more and go to the next segment. + */ + rc = EVP_update(ctx, block, &out_len, + in_addr, len); + if (odp_unlikely(rc != 1)) + goto err; + if (odp_unlikely(out_len > 0)) + goto err; + in_pos += len; + in_len -= len; + buffered += len; + continue; + } } - - /* Use temporal storage */ - if (inseglen > 0) { - unsigned part = inseglen; - - EVP_DecryptUpdate(ctx, block, &cipher_len, - insegaddr, part); - in_pos += part; - in_len -= part; - insegaddr += part; - inseglen -= part; - - odp_packet_copy_from_mem(pkt, out_pos, - cipher_len, block); - out_pos += cipher_len; + ODP_ASSERT(buffered == 0); + + if (in_len > 0) { + /* + * No input is buffered inside OpenSSL. We pass the + * whole remaining segment to OpenSSL and expect to + * get a multiple of block size of data processed, + * with the rest left in the buffer. + */ + rc = EVP_update(ctx, in_addr, &out_len, in_addr, len); + if (odp_unlikely(rc != 1)) + goto err; + ODP_ASSERT(CHECK_IS_POWER2(block_len)); + buffered = len & (block_len - 1); + if (odp_unlikely(out_len + buffered != len)) + goto err; + in_pos += len; + in_len -= len; + out_pos += len - buffered; } } + if (odp_unlikely(buffered > 0)) + goto err; + /* + * We do not expect any more data out since the cipher range is + * supposed to be a multiple of the block size. + */ + rc = EVP_final(ctx, block, &out_len); + if (odp_unlikely(out_len != 0)) + return 0; + return rc; +err: + ODP_ERR("internal error\n"); + (void)EVP_final(ctx, block, &out_len); + return 0; +} - ret = EVP_DecryptFinal_ex(ctx, block, &cipher_len); - odp_packet_copy_from_mem(pkt, out_pos, cipher_len, block); +static int internal_encrypt(EVP_CIPHER_CTX *ctx, + odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param) +{ + return internal_crypt(ctx, pkt, param, + EVP_EncryptUpdate, + EVP_EncryptFinal_ex); +} - return ret; +static int internal_decrypt(EVP_CIPHER_CTX *ctx, + odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param) +{ + return internal_crypt(ctx, pkt, param, + EVP_DecryptUpdate, + EVP_DecryptFinal_ex); } static void @@ -1121,6 +1152,10 @@ odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt, uint32_t in_len = (param->cipher_range.length + 7) / 8; uint8_t data[in_len]; int ret; + uint32_t offset; + + /* Range offset is in bits in bit mode but must be divisible by 8. */ + offset = param->cipher_range.offset / 8; if (param->cipher_iv_ptr) iv_ptr = param->cipher_iv_ptr; @@ -1131,16 +1166,14 @@ odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt, EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - odp_packet_copy_to_mem(pkt, param->cipher_range.offset, in_len, - data); + odp_packet_copy_to_mem(pkt, offset, in_len, data); EVP_EncryptUpdate(ctx, data, &cipher_len, data, in_len); ret = EVP_EncryptFinal_ex(ctx, data + cipher_len, &dummy_len); cipher_len += dummy_len; - odp_packet_copy_from_mem(pkt, param->cipher_range.offset, in_len, - data); + odp_packet_copy_from_mem(pkt, offset, in_len, data); return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; @@ -1159,6 +1192,10 @@ odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt, uint32_t in_len = (param->cipher_range.length + 7) / 8; uint8_t data[in_len]; int ret; + uint32_t offset; + + /* Range offset is in bits in bit mode but must be divisible by 8. */ + offset = param->cipher_range.offset / 8; if (param->cipher_iv_ptr) iv_ptr = param->cipher_iv_ptr; @@ -1169,16 +1206,14 @@ odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt, EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - odp_packet_copy_to_mem(pkt, param->cipher_range.offset, in_len, - data); + odp_packet_copy_to_mem(pkt, offset, in_len, data); EVP_DecryptUpdate(ctx, data, &cipher_len, data, in_len); ret = EVP_DecryptFinal_ex(ctx, data + cipher_len, &dummy_len); cipher_len += dummy_len; - odp_packet_copy_from_mem(pkt, param->cipher_range.offset, in_len, - data); + odp_packet_copy_from_mem(pkt, offset, in_len, data); return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; @@ -2461,12 +2496,18 @@ odp_crypto_operation(odp_crypto_op_param_t *param, packet_param.auth_range = param->auth_range; rc = odp_crypto_op(¶m->pkt, &out_pkt, &packet_param, 1); - if (rc < 0) - return rc; + if (rc <= 0) + return -1; rc = odp_crypto_result(&packet_result, out_pkt); - if (rc < 0) - return rc; + if (rc < 0) { + /* + * We cannot fail since odp_crypto_op() has already processed + * the packet. Let's indicate error in the result instead. + */ + packet_hdr(out_pkt)->p.flags.crypto_err = 1; + packet_result.ok = false; + } /* Indicate to caller operation was sync */ *posted = 0; @@ -2524,7 +2565,7 @@ int _odp_crypto_init_global(void) mem_size += nlocks * sizeof(odp_ticketlock_t); /* Allocate our globally shared memory */ - shm = odp_shm_reserve("_odp_crypto_pool_ssl", mem_size, + shm = odp_shm_reserve("_odp_crypto_ssl_global", mem_size, ODP_CACHE_LINE_SIZE, 0); if (ODP_SHM_INVALID == shm) { @@ -2579,7 +2620,7 @@ int _odp_crypto_term_global(void) CRYPTO_set_id_callback(NULL); #endif - ret = odp_shm_free(odp_shm_lookup("_odp_crypto_pool_ssl")); + ret = odp_shm_free(odp_shm_lookup("_odp_crypto_ssl_global")); if (ret < 0) { ODP_ERR("shm free failed for crypto_pool\n"); rc = -1; diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index 0cb60cdfa..d5530c1f1 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -66,6 +66,7 @@ static void disable_features(odp_global_data_ro_t *global_ro, if (disable_ipsec && disable_crypto) global_ro->disable.crypto = 1; + global_ro->disable.stash = init_param->not_used.feat.stash; global_ro->disable.traffic_mngr = init_param->not_used.feat.tm; global_ro->disable.compress = init_param->not_used.feat.compress; } diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 137e7b435..09596b502 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -20,6 +20,7 @@ #include <odp/api/plat/queue_inlines.h> #include <odp_classification_internal.h> #include <odp_libconfig_internal.h> +#include <odp_schedule_if.h> #include <protocols/eth.h> #include <protocols/ip.h> @@ -164,6 +165,9 @@ int odp_ipsec_capability(odp_ipsec_capability_t *capa) if (rc < 0) return rc; + capa->queue_type_plain = true; + capa->queue_type_sched = true; + rc = odp_queue_capability(&queue_capa); if (rc < 0) return rc; @@ -756,7 +760,8 @@ static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, odp_ipsec_sa_t sa, odp_packet_t *pkt_out, odp_bool_t enqueue_op, - odp_ipsec_op_status_t *status) + odp_ipsec_op_status_t *status, + uint32_t *orig_ip_len) { ipsec_state_t state; ipsec_sa_t *ipsec_sa = NULL; @@ -792,6 +797,7 @@ static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, status->error.alg = 1; goto exit; } + *orig_ip_len = state.ip_tot_len; /* Check IP header for IPSec protocols and look it up */ if (_ODP_IPPROTO_ESP == state.ip_next_hdr || @@ -1014,13 +1020,14 @@ static int ipsec_out_tunnel_parse_ipv6(ipsec_state_t *state, ipsec_sa_t *ipsec_sa) { _odp_ipv6hdr_t *ipv6hdr = state->ip; + uint32_t ver_tc_flow = odp_be_to_cpu_32(ipv6hdr->ver_tc_flow); ipv6hdr->hop_limit -= ipsec_sa->dec_ttl; - state->out_tunnel.ip_tos = (ipv6hdr->ver_tc_flow & + state->out_tunnel.ip_tos = (ver_tc_flow & _ODP_IPV6HDR_TC_MASK) >> _ODP_IPV6HDR_TC_SHIFT; state->out_tunnel.ip_df = 0; - state->out_tunnel.ip_flabel = (ipv6hdr->ver_tc_flow & + state->out_tunnel.ip_flabel = (ver_tc_flow & _ODP_IPV6HDR_FLOW_LABEL_MASK) >> _ODP_IPV6HDR_FLOW_LABEL_SHIFT; state->ip_next_hdr = ipv6hdr->next_hdr; @@ -1740,6 +1747,7 @@ int odp_ipsec_in(const odp_packet_t pkt_in[], int num_in, odp_ipsec_op_status_t status; odp_ipsec_sa_t sa; ipsec_sa_t *ipsec_sa; + uint32_t dummy; /* orig_ip_len not valid in sync operations */ odp_ipsec_packet_result_t *result; memset(&status, 0, sizeof(status)); @@ -1751,7 +1759,7 @@ int odp_ipsec_in(const odp_packet_t pkt_in[], int num_in, ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); } - ipsec_sa = ipsec_in_single(pkt, sa, &pkt, false, &status); + ipsec_sa = ipsec_in_single(pkt, sa, &pkt, false, &status, &dummy); packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); result = ipsec_pkt_result(pkt); @@ -1851,8 +1859,10 @@ int odp_ipsec_in_enq(const odp_packet_t pkt_in[], int num_in, odp_ipsec_op_status_t status; odp_ipsec_sa_t sa; ipsec_sa_t *ipsec_sa; + uint32_t orig_ip_len = 0; odp_ipsec_packet_result_t *result; odp_queue_t queue; + int rc; memset(&status, 0, sizeof(status)); @@ -1863,12 +1873,13 @@ int odp_ipsec_in_enq(const odp_packet_t pkt_in[], int num_in, ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); } - ipsec_sa = ipsec_in_single(pkt, sa, &pkt, true, &status); + ipsec_sa = ipsec_in_single(pkt, sa, &pkt, true, &status, &orig_ip_len); packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); result = ipsec_pkt_result(pkt); memset(result, 0, sizeof(*result)); result->status = status; + result->orig_ip_len = orig_ip_len; if (NULL != ipsec_sa) { result->sa = ipsec_sa->ipsec_sa_hdl; queue = ipsec_sa->queue; @@ -1877,10 +1888,6 @@ int odp_ipsec_in_enq(const odp_packet_t pkt_in[], int num_in, queue = ipsec_config->inbound.default_queue; } - if (odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt))) { - odp_packet_free(pkt); - break; - } in_pkt++; sa_idx += sa_inc; @@ -1891,6 +1898,12 @@ int odp_ipsec_in_enq(const odp_packet_t pkt_in[], int num_in, */ if (sa == ODP_IPSEC_SA_INVALID && ipsec_sa) _odp_ipsec_sa_unuse(ipsec_sa); + + rc = odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt)); + if (odp_unlikely(rc)) { + odp_packet_free(pkt); + break; + } } return in_pkt; @@ -1915,6 +1928,7 @@ int odp_ipsec_out_enq(const odp_packet_t pkt_in[], int num_in, odp_ipsec_packet_result_t *result; const odp_ipsec_out_opt_t *opt; odp_queue_t queue; + int rc; memset(&status, 0, sizeof(status)); @@ -1939,13 +1953,15 @@ int odp_ipsec_out_enq(const odp_packet_t pkt_in[], int num_in, if (ipsec_config->stats_en) ipsec_sa_err_stats_update(ipsec_sa, &status); - if (odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt))) { - odp_packet_free(pkt); - break; - } in_pkt++; sa_idx += sa_inc; opt_idx += opt_inc; + + rc = odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt)); + if (odp_unlikely(rc)) { + odp_packet_free(pkt); + break; + } } return in_pkt; @@ -1955,6 +1971,7 @@ int _odp_ipsec_try_inline(odp_packet_t *pkt) { odp_ipsec_op_status_t status; ipsec_sa_t *ipsec_sa; + uint32_t orig_ip_len = 0; odp_ipsec_packet_result_t *result; odp_packet_hdr_t *pkt_hdr; @@ -1964,7 +1981,7 @@ int _odp_ipsec_try_inline(odp_packet_t *pkt) memset(&status, 0, sizeof(status)); ipsec_sa = ipsec_in_single(*pkt, ODP_IPSEC_SA_INVALID, pkt, false, - &status); + &status, &orig_ip_len); /* * Route packet back in case of lookup failure or early error before * lookup @@ -1976,6 +1993,7 @@ int _odp_ipsec_try_inline(odp_packet_t *pkt) result = ipsec_pkt_result(*pkt); memset(result, 0, sizeof(*result)); result->status = status; + result->orig_ip_len = orig_ip_len; result->sa = ipsec_sa->ipsec_sa_hdl; result->flag.inline_mode = 1; @@ -2096,6 +2114,7 @@ int odp_ipsec_out_inline(const odp_packet_t pkt_in[], int num_in, } } else { odp_queue_t queue; + int rc; err: if (ipsec_config->stats_en) ipsec_sa_err_stats_update(ipsec_sa, &status); @@ -2107,8 +2126,9 @@ err: result->status = status; queue = ipsec_sa->queue; - if (odp_queue_enq(queue, - odp_ipsec_packet_to_event(pkt))) { + rc = odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt)); + if (odp_unlikely(rc)) { + in_pkt++; odp_packet_free(pkt); break; } @@ -2196,6 +2216,21 @@ int odp_ipsec_stats(odp_ipsec_sa_t sa, odp_ipsec_stats_t *stats) return 0; } +int odp_ipsec_stats_multi(odp_ipsec_sa_t sa[], odp_ipsec_stats_t stats[], int num) +{ + int ret, i; + + ODP_ASSERT(NULL != stats); + + for (i = 0; i < num; i++) { + ret = odp_ipsec_stats(sa[i], &stats[i]); + if (ret) + return ret; + } + + return 0; +} + static int read_config_file(ipsec_global_t *global) { const char *str_i = "ipsec.ordering.async_inbound"; @@ -2224,7 +2259,7 @@ int _odp_ipsec_init_global(void) if (odp_global_ro.disable.ipsec) return 0; - shm = odp_shm_reserve("_odp_ipsec", sizeof(*ipsec_global), + shm = odp_shm_reserve("_odp_ipsec_global", sizeof(*ipsec_global), ODP_CACHE_LINE_SIZE, 0); if (shm == ODP_SHM_INVALID) { ODP_ERR("Shm reserve failed for odp_ipsec\n"); @@ -2260,7 +2295,7 @@ int _odp_ipsec_term_global(void) if (odp_global_ro.disable.ipsec) return 0; - shm = odp_shm_lookup("_odp_ipsec"); + shm = odp_shm_lookup("_odp_ipsec_global"); if (shm == ODP_SHM_INVALID || odp_shm_free(shm)) { ODP_ERR("Shm free failed for odp_ipsec"); diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 7041cd2b5..ebc931658 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -75,7 +75,7 @@ * if some of the block owners never procsync() after free). This number * should take that into account) */ -#define ISHM_MAX_NB_BLOCKS 128 +#define ISHM_MAX_NB_BLOCKS (CONFIG_INTERNAL_SHM_BLOCKS + CONFIG_SHM_BLOCKS) /* * Maximum internal shared memory block name length in chars diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 0986056e6..c6a50bf84 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -11,6 +11,7 @@ #include <odp/api/plat/packet_inlines.h> #include <odp_packet_internal.h> #include <odp_debug_internal.h> +#include <odp_macros_internal.h> #include <odp_chksum_internal.h> #include <odp_errno_define.h> #include <odp/api/hints.h> @@ -1847,7 +1848,15 @@ static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr, return sum; } -/** Parser helper function for Ethernet packets */ +/* + * In the worst case we look at the Ethernet header, 8 bytes of LLC/SNAP + * header and two VLAN tags in the same packet. + */ +#define PARSE_ETH_BYTES (sizeof(_odp_ethhdr_t) + 8 + 2 * sizeof(_odp_vlanhdr_t)) +/** Parser helper function for Ethernet packets + * + * Requires up to PARSE_ETH_BYTES bytes of contiguous packet data. + */ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len) { @@ -1864,7 +1873,7 @@ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, eth = (const _odp_ethhdr_t *)*parseptr; /* Detect jumbo frames */ - if (odp_unlikely(frame_len > _ODP_ETH_LEN_MAX)) + if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX)) input_flags.jumbo = 1; /* Handle Ethernet broadcast/multicast addresses */ @@ -1922,14 +1931,27 @@ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_vlanhdr_t); } + /* + * The packet was too short for what we parsed. We just give up + * entirely without trying to parse what fits in the packet. + */ + if (odp_unlikely(*offset > frame_len)) { + input_flags.all = 0; + input_flags.l2 = 1; + ethtype = 0; + } + error: prs->input_flags.all |= input_flags.all; return ethtype; } +#define PARSE_IPV4_BYTES (0xfU * 4) /* max IPv4 header length with options */ /** * Parser helper function for IPv4 + * + * Requires up to PARSE_IPV4_BYTES bytes of contiguous packet data. */ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, @@ -1945,6 +1967,7 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN || ver != 4 || + sizeof(*ipv4) > frame_len - *offset || (l3_len > frame_len - *offset))) { prs->flags.ip_err = 1; return 0; @@ -1987,8 +2010,15 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, return ipv4->proto; } +/* + * Peeks 2 bytes beyond IPv6 base header without length check if there + * are extension headers. + */ +#define PARSE_IPV6_BYTES (sizeof(_odp_ipv6hdr_t) + 2) /** * Parser helper function for IPv6 + * + * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data. */ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, @@ -2003,8 +2033,9 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, _ODP_IPV6HDR_LEN; /* Basic sanity checks on IPv6 header */ - if ((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || - l3_len > frame_len - *offset) { + if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || + sizeof(*ipv6) > frame_len - *offset || + l3_len > frame_len - *offset)) { prs->flags.ip_err = 1; return 0; } @@ -2056,8 +2087,11 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, return ipv6->next_hdr; } +#define PARSE_TCP_BYTES (sizeof(_odp_tcphdr_t)) /** * Parser helper function for TCP + * + * Requires PARSE_TCP_BYTES bytes of contiguous packet data. */ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t tcp_len, @@ -2083,8 +2117,15 @@ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += len; } +/* + * In the worst case we look at the UDP header and 4 bytes of the UDP + * payload (the non-ESP marker to distinguish IKE packets from ESP packets). + */ +#define PARSE_UDP_BYTES (sizeof(_odp_udphdr_t) + 4) /** * Parser helper function for UDP + * + * Requires PARSE_UDP_BYTES bytes of contiguous packet data. */ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, odp_proto_chksums_t chksums, @@ -2129,8 +2170,11 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_udphdr_t); } +#define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t)) /** * Parser helper function for SCTP + * + * Requires PARSE_SCTP_BYTES bytes of contiguous packet data. */ static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t sctp_len, @@ -2157,6 +2201,10 @@ static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_sctphdr_t); } +#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c))) +#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \ + MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES)) +/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */ static inline int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, uint32_t offset, @@ -2516,6 +2564,8 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, if (chksums.chksum.sctp && pkt_hdr->p.input_flags.sctp && !pkt_hdr->p.input_flags.ipfrag) { + uint32_t seg_len = 0; + _odp_sctphdr_t hdr_copy; uint32_t sum = ~packet_sum_crc32c(pkt_hdr, pkt_hdr->p.l4_offset + _ODP_SCTPHDR_LEN, @@ -2525,8 +2575,14 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, l4_part_sum); _odp_sctphdr_t *sctp = packet_map(pkt_hdr, pkt_hdr->p.l4_offset, - NULL, NULL); + &seg_len, NULL); + if (odp_unlikely(seg_len < sizeof(*sctp))) { + odp_packet_t pkt = packet_handle(pkt_hdr); + sctp = &hdr_copy; + odp_packet_copy_to_mem(pkt, pkt_hdr->p.l4_offset, + sizeof(*sctp), sctp); + } pkt_hdr->p.input_flags.l4_chksum_done = 1; if (sum != sctp->chksum) { pkt_hdr->p.flags.l4_chksum_err = 1; @@ -2587,6 +2643,8 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, int ret; uint16_t ethtype; uint64_t l4_part_sum = 0; + const uint32_t min_seglen = PARSE_ETH_BYTES + PARSE_L3_L4_BYTES; + uint8_t buf[min_seglen]; if (proto == ODP_PROTO_NONE || layer == ODP_PROTO_LAYER_NONE) return -1; @@ -2596,6 +2654,20 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, if (data == NULL) return -1; + /* + * We must not have a packet segment boundary within the parsed + * packet data range. Copy enough data to a temporary buffer for + * parsing if necessary. + */ + if (odp_unlikely(pkt_hdr->seg_count > 1) && + odp_unlikely(seg_len < min_seglen)) { + seg_len = min_seglen; + if (seg_len > packet_len - offset) + seg_len = packet_len - offset; + odp_packet_copy_to_mem(pkt, offset, seg_len, buf); + data = buf; + } + /* Reset parser flags, keep other flags */ packet_parse_reset(pkt_hdr, 0); @@ -2942,6 +3014,13 @@ odp_packet_reass_status(odp_packet_t pkt) return ODP_PACKET_REASS_NONE; } +int odp_packet_reass_info(odp_packet_t pkt, odp_packet_reass_info_t *info) +{ + (void)pkt; + (void)info; + return -1; +} + int odp_packet_reass_partial_state(odp_packet_t pkt, odp_packet_t frags[], odp_packet_reass_partial_state_t *res) diff --git a/platform/linux-generic/odp_pcapng.c b/platform/linux-generic/odp_pcapng.c index 68a18b305..ce950760e 100644 --- a/platform/linux-generic/odp_pcapng.c +++ b/platform/linux-generic/odp_pcapng.c @@ -90,7 +90,7 @@ int _odp_pcapng_init_global(void) { odp_shm_t shm; - shm = odp_shm_reserve("_odp_pcapng_gbl", sizeof(pcapng_global_t), + shm = odp_shm_reserve("_odp_pcapng_global", sizeof(pcapng_global_t), ODP_PAGE_SIZE, 0); if (shm == ODP_SHM_INVALID) return -1; diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index d1fc94369..9c60a9458 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -271,7 +271,7 @@ int _odp_pool_init_global(void) uint32_t i; odp_shm_t shm; - shm = odp_shm_reserve("_odp_pool_table", + shm = odp_shm_reserve("_odp_pool_global", sizeof(pool_global_t), ODP_CACHE_LINE_SIZE, 0); diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 428022ec8..e35f01f7d 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -132,7 +132,7 @@ static int queue_init_global(void) _odp_queue_inline_offset.context = offsetof(queue_entry_t, s.param.context); - shm = odp_shm_reserve("_odp_queue_gbl", + shm = odp_shm_reserve("_odp_queue_basic_global", sizeof(queue_global_t), sizeof(queue_entry_t), 0); @@ -161,7 +161,7 @@ static int queue_init_global(void) mem_size = sizeof(uint32_t) * CONFIG_MAX_QUEUES * (uint64_t)_odp_queue_glb->config.max_queue_size; - shm = odp_shm_reserve("_odp_queue_rings", mem_size, + shm = odp_shm_reserve("_odp_queue_basic_rings", mem_size, ODP_CACHE_LINE_SIZE, 0); @@ -677,7 +677,8 @@ static void queue_print(odp_queue_t handle) odp_pktio_info_t pktio_info; queue_entry_t *queue; uint32_t queue_id; - int status; + int status, prio; + int max_prio = odp_schedule_max_prio(); queue_id = queue_to_index(handle); @@ -701,7 +702,7 @@ static void queue_print(odp_queue_t handle) ODP_PRINT("\nQueue info\n"); ODP_PRINT("----------\n"); ODP_PRINT(" handle %p\n", queue->s.handle); - ODP_PRINT(" index %" PRIu32 "\n", queue->s.index); + ODP_PRINT(" index %" PRIu32 "\n", queue_id); ODP_PRINT(" name %s\n", queue->s.name); ODP_PRINT(" enq mode %s\n", queue->s.param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : @@ -729,8 +730,11 @@ static void queue_print(odp_queue_t handle) "ODP_SCHED_SYNC_ATOMIC" : (queue->s.param.sched.sync == ODP_SCHED_SYNC_ORDERED ? "ODP_SCHED_SYNC_ORDERED" : "unknown"))); - ODP_PRINT(" priority %d\n", queue->s.param.sched.prio); - ODP_PRINT(" group %d\n", queue->s.param.sched.group); + prio = queue->s.param.sched.prio; + ODP_PRINT(" priority %i (%i in API)\n", max_prio - prio, prio); + ODP_PRINT(" group %i\n", queue->s.param.sched.group); + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + ODP_PRINT(" spread %i\n", _odp_sched_basic_get_spread(queue_id)); } if (queue->s.pktin.pktio != ODP_PKTIO_INVALID) { if (!odp_pktio_info(queue->s.pktin.pktio, &pktio_info)) @@ -784,11 +788,17 @@ static void queue_print_all(void) char type_c, enq_c, deq_c, order_c, sync_c; const int col_width = 24; int prio = 0; + int spr = 0; odp_schedule_sync_t sync = ODP_SCHED_SYNC_PARALLEL; + odp_schedule_group_t grp = ODP_SCHED_GROUP_INVALID; ODP_PRINT("\nList of all queues\n"); ODP_PRINT("------------------\n"); - ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio\n", col_width, "name"); + ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio grp", col_width, "name"); + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + ODP_PRINT(" spr\n"); + else + ODP_PRINT("\n"); for (i = 0; i < CONFIG_MAX_QUEUES; i++) { queue_entry_t *queue = qentry_from_index(i); @@ -817,7 +827,10 @@ static void queue_print_all(void) len = ring_st_length(&queue->s.ring_st); max_len = queue->s.ring_mask + 1; prio = queue->s.param.sched.prio; + grp = queue->s.param.sched.group; sync = queue->s.param.sched.sync; + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + spr = _odp_sched_basic_get_spread(index); } else { len = ring_mpmc_length(&queue->s.ring_mpmc); max_len = queue->s.ring_mask + 1; @@ -851,7 +864,13 @@ static void queue_print_all(void) if (type == ODP_QUEUE_TYPE_SCHED) { sync_c = (sync == ODP_SCHED_SYNC_PARALLEL) ? 'P' : ((sync == ODP_SCHED_SYNC_ATOMIC) ? 'A' : 'O'); - ODP_PRINT(" %c %4i", sync_c, prio); + /* Print prio level matching odp_schedule_print() output */ + prio = odp_schedule_max_prio() - prio; + + ODP_PRINT(" %c %4i %3i", sync_c, prio, grp); + + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + ODP_PRINT(" %3i", spr); } ODP_PRINT("\n"); diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c index 70d555ab5..206172bb8 100644 --- a/platform/linux-generic/odp_queue_lf.c +++ b/platform/linux-generic/odp_queue_lf.c @@ -268,7 +268,7 @@ uint32_t _odp_queue_lf_init_global(uint32_t *queue_lf_size, if (!lockfree) return 0; - shm = odp_shm_reserve("_odp_queues_lf", sizeof(queue_lf_global_t), + shm = odp_shm_reserve("_odp_queues_lf_global", sizeof(queue_lf_global_t), ODP_CACHE_LINE_SIZE, 0); if (shm == ODP_SHM_INVALID) diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 479d6f956..eb17e95eb 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -78,6 +78,9 @@ ODP_STATIC_ASSERT((QUEUE_LOAD * CONFIG_MAX_SCHED_QUEUES) < UINT32_MAX, "Load_val /* Spread weight table */ #define SPREAD_TBL_SIZE ((MAX_SPREAD - 1) * MAX_PREFER_RATIO) +/* Random data table size */ +#define RANDOM_TBL_SIZE 128 + /* Maximum number of packet IO interfaces */ #define NUM_PKTIO ODP_CONFIG_PKTIO_ENTRIES @@ -133,6 +136,27 @@ typedef union { ODP_STATIC_ASSERT(sizeof(lock_called_t) == sizeof(uint32_t), "Lock_called_values_do_not_fit_in_uint32"); +static uint8_t sched_random_u8[] = { + 0x64, 0xe3, 0x64, 0x0a, 0x0a, 0x5b, 0x7e, 0xd7, + 0x43, 0xb7, 0x90, 0x71, 0x76, 0x17, 0x8e, 0x3f, + 0x17, 0x60, 0x7e, 0xfd, 0x99, 0xe3, 0xab, 0x06, + 0x77, 0xf9, 0x45, 0x17, 0x2f, 0x81, 0x9e, 0x7b, + 0x20, 0x1b, 0x36, 0x75, 0x69, 0xc5, 0x69, 0x27, + 0x7a, 0xf6, 0x3f, 0x63, 0x2c, 0x3f, 0x1b, 0xeb, + 0x12, 0xe1, 0x6f, 0xd4, 0xd9, 0x14, 0x97, 0xa6, + 0x2a, 0xe5, 0xb0, 0x45, 0x27, 0xa6, 0x48, 0xbc, + 0x2b, 0xec, 0xd8, 0xda, 0x55, 0xef, 0x15, 0xce, + 0xf8, 0xc2, 0x1e, 0xc8, 0x16, 0x6c, 0xf0, 0x4f, + 0x1a, 0xc7, 0x50, 0x9e, 0x0b, 0xa5, 0xe9, 0xf3, + 0x28, 0x79, 0x2e, 0x18, 0xb0, 0xb4, 0xac, 0xce, + 0x67, 0x04, 0x52, 0x98, 0xce, 0x8c, 0x05, 0x87, + 0xab, 0xc8, 0x94, 0x7e, 0x46, 0x63, 0x60, 0x8d, + 0x3d, 0x8f, 0x14, 0x85, 0x1e, 0x92, 0xd2, 0x40, + 0x2d, 0x42, 0xfe, 0xf1, 0xc2, 0xb6, 0x03, 0x43 +}; + +ODP_STATIC_ASSERT(sizeof(sched_random_u8) == RANDOM_TBL_SIZE, "Bad_random_table_size"); + /* Scheduler local data */ typedef struct ODP_ALIGNED_CACHE { uint32_t sched_round; @@ -247,6 +271,7 @@ typedef struct { /* Scheduler interface config options (not used in fast path) */ schedule_config_t config_if; uint32_t max_queues; + odp_atomic_u32_t next_rand; } sched_global_t; @@ -438,7 +463,7 @@ static int schedule_init_global(void) ODP_DBG("Schedule init ... "); - shm = odp_shm_reserve("_odp_scheduler", + shm = odp_shm_reserve("_odp_sched_basic_global", sizeof(sched_global_t), ODP_CACHE_LINE_SIZE, 0); @@ -500,6 +525,7 @@ static int schedule_init_global(void) odp_ticketlock_init(&sched->grp_lock); odp_atomic_init_u32(&sched->grp_epoch, 0); + odp_atomic_init_u32(&sched->next_rand, 0); for (i = 0; i < NUM_SCHED_GRPS; i++) { memset(sched->sched_grp[i].name, 0, ODP_SCHED_GROUP_NAME_LEN); @@ -662,25 +688,42 @@ static inline void update_queue_count(int grp, int prio, int old_spr, int new_sp odp_ticketlock_unlock(&sched->mask_lock[grp]); } -/* Select the spread that has least queues */ static uint8_t allocate_spread(int grp, int prio) { - uint8_t i; + uint8_t i, num_min, spr; uint32_t num; uint32_t min = UINT32_MAX; uint8_t num_spread = sched->config.num_spread; - uint8_t spr = 0; + uint8_t min_spr[num_spread]; + + num_min = 1; + min_spr[0] = 0; odp_ticketlock_lock(&sched->mask_lock[grp]); + /* Find spread(s) with the minimum number of queues */ for (i = 0; i < num_spread; i++) { num = sched->prio_q_count[grp][prio][i]; if (num < min) { - spr = i; min = num; + min_spr[0] = i; + num_min = 1; + } else if (num == min) { + min_spr[num_min] = i; + num_min++; } } + spr = min_spr[0]; + + /* When there are multiple minimum spreads, select one randomly */ + if (num_min > 1) { + uint32_t next_rand = odp_atomic_fetch_inc_u32(&sched->next_rand); + uint8_t rand = sched_random_u8[next_rand % RANDOM_TBL_SIZE]; + + spr = min_spr[rand % num_min]; + } + sched->prio_q_mask[grp][prio] |= 1 << spr; sched->prio_q_count[grp][prio][spr]++; @@ -719,6 +762,16 @@ static int schedule_create_queue(uint32_t queue_index, return -1; } + odp_ticketlock_lock(&sched->grp_lock); + + if (sched->sched_grp[grp].allocated == 0) { + odp_ticketlock_unlock(&sched->grp_lock); + ODP_ERR("Group not created: %i\n", grp); + return -1; + } + + odp_ticketlock_unlock(&sched->grp_lock); + spread = allocate_spread(grp, prio); sched->queue[queue_index].grp = grp; @@ -1573,11 +1626,64 @@ static uint64_t schedule_wait_time(uint64_t ns) return ns; } +static inline void spread_thrs_inc(odp_schedule_group_t group, int thr_tbl[], int count) +{ + int thr, i; + uint8_t spread; + + for (i = 0; i < count; i++) { + thr = thr_tbl[i]; + spread = spread_from_index(thr); + sched->sched_grp[group].spread_thrs[spread]++; + } +} + +static inline void spread_thrs_dec(odp_schedule_group_t group, int thr_tbl[], int count) +{ + int thr, i; + uint8_t spread; + + for (i = 0; i < count; i++) { + thr = thr_tbl[i]; + spread = spread_from_index(thr); + sched->sched_grp[group].spread_thrs[spread]--; + } +} + +static inline int threads_from_mask(int thr_tbl[], int count, const odp_thrmask_t *mask) +{ + int i; + int thr = odp_thrmask_first(mask); + + for (i = 0; i < count; i++) { + if (thr < 0) { + ODP_ERR("No more threads in the mask\n"); + return -1; + } + + thr_tbl[i] = thr; + thr = odp_thrmask_next(mask, thr); + } + + return 0; +} + static odp_schedule_group_t schedule_group_create(const char *name, const odp_thrmask_t *mask) { odp_schedule_group_t group = ODP_SCHED_GROUP_INVALID; - int i; + int count, i; + + count = odp_thrmask_count(mask); + if (count < 0) { + ODP_ERR("Bad thread count\n"); + return ODP_SCHED_GROUP_INVALID; + } + + int thr_tbl[count]; + + if (count && threads_from_mask(thr_tbl, count, mask)) + return ODP_SCHED_GROUP_INVALID; odp_ticketlock_lock(&sched->grp_lock); @@ -1595,6 +1701,7 @@ static odp_schedule_group_t schedule_group_create(const char *name, grp_update_mask(i, mask); group = (odp_schedule_group_t)i; + spread_thrs_inc(group, thr_tbl, count); sched->sched_grp[i].allocated = 1; break; } @@ -1607,25 +1714,33 @@ static odp_schedule_group_t schedule_group_create(const char *name, static int schedule_group_destroy(odp_schedule_group_t group) { odp_thrmask_t zero; - int ret; + int i; + + if (group >= NUM_SCHED_GRPS || group < SCHED_GROUP_NAMED) { + ODP_ERR("Bad group %i\n", group); + return -1; + } odp_thrmask_zero(&zero); odp_ticketlock_lock(&sched->grp_lock); - if (group < NUM_SCHED_GRPS && group >= SCHED_GROUP_NAMED && - sched->sched_grp[group].allocated) { - grp_update_mask(group, &zero); - memset(sched->sched_grp[group].name, 0, - ODP_SCHED_GROUP_NAME_LEN); - sched->sched_grp[group].allocated = 0; - ret = 0; - } else { - ret = -1; + if (sched->sched_grp[group].allocated == 0) { + odp_ticketlock_unlock(&sched->grp_lock); + ODP_ERR("Group not created: %i\n", group); + return -1; } + grp_update_mask(group, &zero); + + for (i = 0; i < MAX_SPREAD; i++) + sched->sched_grp[group].spread_thrs[i] = 0; + + memset(sched->sched_grp[group].name, 0, ODP_SCHED_GROUP_NAME_LEN); + sched->sched_grp[group].allocated = 0; + odp_ticketlock_unlock(&sched->grp_lock); - return ret; + return 0; } static odp_schedule_group_t schedule_group_lookup(const char *name) @@ -1649,7 +1764,6 @@ static odp_schedule_group_t schedule_group_lookup(const char *name) static int schedule_group_join(odp_schedule_group_t group, const odp_thrmask_t *mask) { int i, count, thr; - uint8_t spread; odp_thrmask_t new_mask; if (group >= NUM_SCHED_GRPS || group < SCHED_GROUP_NAMED) { @@ -1684,10 +1798,7 @@ static int schedule_group_join(odp_schedule_group_t group, const odp_thrmask_t * return -1; } - for (i = 0; i < count; i++) { - spread = spread_from_index(thr_tbl[i]); - sched->sched_grp[group].spread_thrs[spread]++; - } + spread_thrs_inc(group, thr_tbl, count); odp_thrmask_or(&new_mask, &sched->sched_grp[group].mask, mask); grp_update_mask(group, &new_mask); @@ -1699,7 +1810,6 @@ static int schedule_group_join(odp_schedule_group_t group, const odp_thrmask_t * static int schedule_group_leave(odp_schedule_group_t group, const odp_thrmask_t *mask) { int i, count, thr; - uint8_t spread; odp_thrmask_t new_mask; if (group >= NUM_SCHED_GRPS || group < SCHED_GROUP_NAMED) { @@ -1736,10 +1846,7 @@ static int schedule_group_leave(odp_schedule_group_t group, const odp_thrmask_t return -1; } - for (i = 0; i < count; i++) { - spread = spread_from_index(thr_tbl[i]); - sched->sched_grp[group].spread_thrs[spread]--; - } + spread_thrs_dec(group, thr_tbl, count); odp_thrmask_and(&new_mask, &sched->sched_grp[group].mask, &new_mask); grp_update_mask(group, &new_mask); @@ -1789,7 +1896,6 @@ static int schedule_thr_add(odp_schedule_group_t group, int thr) { odp_thrmask_t mask; odp_thrmask_t new_mask; - uint8_t spread = spread_from_index(thr); if (group < 0 || group >= SCHED_GROUP_NAMED) return -1; @@ -1805,7 +1911,7 @@ static int schedule_thr_add(odp_schedule_group_t group, int thr) } odp_thrmask_or(&new_mask, &sched->sched_grp[group].mask, &mask); - sched->sched_grp[group].spread_thrs[spread]++; + spread_thrs_inc(group, &thr, 1); grp_update_mask(group, &new_mask); odp_ticketlock_unlock(&sched->grp_lock); @@ -1817,7 +1923,6 @@ static int schedule_thr_rem(odp_schedule_group_t group, int thr) { odp_thrmask_t mask; odp_thrmask_t new_mask; - uint8_t spread = spread_from_index(thr); if (group < 0 || group >= SCHED_GROUP_NAMED) return -1; @@ -1834,7 +1939,7 @@ static int schedule_thr_rem(odp_schedule_group_t group, int thr) } odp_thrmask_and(&new_mask, &sched->sched_grp[group].mask, &new_mask); - sched->sched_grp[group].spread_thrs[spread]--; + spread_thrs_dec(group, &thr, 1); grp_update_mask(group, &new_mask); odp_ticketlock_unlock(&sched->grp_lock); @@ -1947,6 +2052,12 @@ static void schedule_print(void) ODP_PRINT("\n"); } +/* Returns spread for queue debug prints */ +int _odp_sched_basic_get_spread(uint32_t queue_index) +{ + return sched->queue[queue_index].spread; +} + /* Fill in scheduler interface */ const schedule_fn_t _odp_schedule_basic_fn = { .pktio_start = schedule_pktio_start, diff --git a/platform/linux-generic/odp_schedule_if.c b/platform/linux-generic/odp_schedule_if.c index 24307ab90..528c4065e 100644 --- a/platform/linux-generic/odp_schedule_if.c +++ b/platform/linux-generic/odp_schedule_if.c @@ -1,4 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -25,6 +26,7 @@ extern const schedule_api_t _odp_schedule_scalable_api; const schedule_fn_t *_odp_sched_fn; const schedule_api_t *_odp_sched_api; +int _odp_sched_id; uint64_t odp_schedule_wait_time(uint64_t ns) { @@ -216,12 +218,15 @@ int _odp_schedule_init_global(void) ODP_PRINT("Using scheduler '%s'\n", sched); if (!strcmp(sched, "basic")) { + _odp_sched_id = _ODP_SCHED_ID_BASIC; _odp_sched_fn = &_odp_schedule_basic_fn; _odp_sched_api = &_odp_schedule_basic_api; } else if (!strcmp(sched, "sp")) { + _odp_sched_id = _ODP_SCHED_ID_SP; _odp_sched_fn = &_odp_schedule_sp_fn; _odp_sched_api = &_odp_schedule_sp_api; } else if (!strcmp(sched, "scalable")) { + _odp_sched_id = _ODP_SCHED_ID_SCALABLE; _odp_sched_fn = &_odp_schedule_scalable_fn; _odp_sched_api = &_odp_schedule_scalable_api; } else { diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 18e738330..10f456eed 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -1792,7 +1792,7 @@ static int schedule_init_global(void) uint64_t min_alloc; uint64_t max_alloc; - shm = odp_shm_reserve("_odp_sched_scalable", + shm = odp_shm_reserve("_odp_sched_scalable_global", sizeof(sched_global_t), ODP_CACHE_LINE_SIZE, 0); diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index c4f8344f1..4f6ce736d 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -172,7 +172,7 @@ static int init_global(void) ODP_DBG("Using SP scheduler\n"); - shm = odp_shm_reserve("sp_scheduler", + shm = odp_shm_reserve("_odp_sched_sp_global", sizeof(sched_global_t), ODP_CACHE_LINE_SIZE, 0); diff --git a/platform/linux-generic/odp_shared_memory.c b/platform/linux-generic/odp_shared_memory.c index ee47b7e96..966850a07 100644 --- a/platform/linux-generic/odp_shared_memory.c +++ b/platform/linux-generic/odp_shared_memory.c @@ -15,9 +15,6 @@ #include <odp_global_data.h> #include <string.h> -ODP_STATIC_ASSERT(ODP_CONFIG_SHM_BLOCKS >= ODP_CONFIG_POOLS, - "ODP_CONFIG_SHM_BLOCKS < ODP_CONFIG_POOLS"); - static inline uint32_t from_handle(odp_shm_t shm) { return _odp_typeval(shm) - 1; @@ -47,7 +44,7 @@ int odp_shm_capability(odp_shm_capability_t *capa) { memset(capa, 0, sizeof(odp_shm_capability_t)); - capa->max_blocks = ODP_CONFIG_SHM_BLOCKS; + capa->max_blocks = CONFIG_SHM_BLOCKS; capa->max_size = odp_global_ro.shm_max_size; capa->max_align = 0; diff --git a/platform/linux-generic/odp_stash.c b/platform/linux-generic/odp_stash.c index d2062379b..a1bf5b63d 100644 --- a/platform/linux-generic/odp_stash.c +++ b/platform/linux-generic/odp_stash.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2020, Nokia +/* Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -13,6 +13,7 @@ #include <odp/api/plat/strong_types.h> #include <odp_debug_internal.h> +#include <odp_global_data.h> #include <odp_init_internal.h> #include <odp_ring_ptr_internal.h> #include <odp_ring_u32_internal.h> @@ -57,7 +58,12 @@ int _odp_stash_init_global(void) { odp_shm_t shm; - shm = odp_shm_reserve("_odp_stash_table", sizeof(stash_global_t), + if (odp_global_ro.disable.stash) { + ODP_PRINT("Stash is DISABLED\n"); + return 0; + } + + shm = odp_shm_reserve("_odp_stash_global", sizeof(stash_global_t), ODP_CACHE_LINE_SIZE, 0); stash_global = odp_shm_addr(shm); @@ -76,6 +82,9 @@ int _odp_stash_init_global(void) int _odp_stash_term_global(void) { + if (odp_global_ro.disable.stash) + return 0; + if (stash_global == NULL) return 0; @@ -89,6 +98,11 @@ int _odp_stash_term_global(void) int odp_stash_capability(odp_stash_capability_t *capa, odp_stash_type_t type) { + if (odp_global_ro.disable.stash) { + ODP_ERR("Stash is disabled\n"); + return -1; + } + (void)type; memset(capa, 0, sizeof(odp_stash_capability_t)); @@ -146,6 +160,11 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) int ring_ptr, index; char shm_name[ODP_STASH_NAME_LEN + 8]; + if (odp_global_ro.disable.stash) { + ODP_ERR("Stash is disabled\n"); + return ODP_STASH_INVALID; + } + if (param->obj_size > sizeof(uintptr_t)) { ODP_ERR("Too large object handle.\n"); return ODP_STASH_INVALID; diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index 27c591a9c..778ea08cb 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -579,7 +579,7 @@ void odp_sys_config_print(void) ODP_PRINT("ODP_CONFIG_PKTIO_ENTRIES: %i\n", ODP_CONFIG_PKTIO_ENTRIES); ODP_PRINT("CONFIG_PACKET_HEADROOM: %i\n", CONFIG_PACKET_HEADROOM); ODP_PRINT("CONFIG_PACKET_TAILROOM: %i\n", CONFIG_PACKET_TAILROOM); - ODP_PRINT("ODP_CONFIG_SHM_BLOCKS: %i\n", ODP_CONFIG_SHM_BLOCKS); + ODP_PRINT("CONFIG_SHM_BLOCKS: %i\n", CONFIG_SHM_BLOCKS); ODP_PRINT("CONFIG_BURST_SIZE: %i\n", CONFIG_BURST_SIZE); ODP_PRINT("CONFIG_POOL_MAX_NUM: %i\n", CONFIG_POOL_MAX_NUM); ODP_PRINT("CONFIG_POOL_CACHE_MAX_SIZE: %i\n", CONFIG_POOL_CACHE_MAX_SIZE); diff --git a/platform/linux-generic/odp_thread.c b/platform/linux-generic/odp_thread.c index af891bce8..ffb116974 100644 --- a/platform/linux-generic/odp_thread.c +++ b/platform/linux-generic/odp_thread.c @@ -68,7 +68,7 @@ int _odp_thread_init_global(void) if (num_max > ODP_THREAD_COUNT_MAX) num_max = ODP_THREAD_COUNT_MAX; - shm = odp_shm_reserve("_odp_thread_globals", + shm = odp_shm_reserve("_odp_thread_global", sizeof(thread_globals_t), ODP_CACHE_LINE_SIZE, 0); @@ -97,7 +97,7 @@ int _odp_thread_term_global(void) if (num) ODP_ERR("%u threads have not called odp_term_local().\n", num); - ret = odp_shm_free(odp_shm_lookup("_odp_thread_globals")); + ret = odp_shm_free(odp_shm_lookup("_odp_thread_global")); if (ret < 0) ODP_ERR("shm free failed for _odp_thread_globals"); diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index e5b42a83b..f1ca28346 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -1599,7 +1599,7 @@ int _odp_timer_init_global(const odp_init_t *params) return 0; } - shm = odp_shm_reserve("_odp_timer", sizeof(timer_global_t), + shm = odp_shm_reserve("_odp_timer_global", sizeof(timer_global_t), ODP_CACHE_LINE_SIZE, 0); timer_global = odp_shm_addr(shm); diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index de01af96c..e741fd80c 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -32,6 +32,7 @@ #include <odp_init_internal.h> #include <odp_errno_define.h> #include <odp_global_data.h> +#include <odp_schedule_if.h> /* Local vars */ static const @@ -4935,7 +4936,7 @@ int _odp_tm_init_global(void) return 0; } - shm = odp_shm_reserve("_odp_traffic_mng", sizeof(tm_global_t), 0, 0); + shm = odp_shm_reserve("_odp_traffic_mng_global", sizeof(tm_global_t), 0, 0); if (shm == ODP_SHM_INVALID) return -1; diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index a98324ee4..9abba6292 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -288,7 +288,7 @@ static inline void mbuf_update(struct rte_mbuf *mbuf, odp_packet_hdr_t *pkt_hdr, mbuf->refcnt = 1; mbuf->ol_flags = 0; - if (odp_unlikely(pkt_hdr->buf_hdr.base_data != pkt_hdr->seg_data)) + if (odp_unlikely(((uint8_t *)mbuf->buf_addr + mbuf->data_off) != pkt_hdr->seg_data)) mbuf->data_off = mbuf_data_off(mbuf, pkt_hdr); } diff --git a/platform/linux-generic/pktio/dpdk_parse.c b/platform/linux-generic/pktio/dpdk_parse.c index f593f4b11..c1b75f126 100644 --- a/platform/linux-generic/pktio/dpdk_parse.c +++ b/platform/linux-generic/pktio/dpdk_parse.c @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -53,7 +54,7 @@ static inline uint16_t dpdk_parse_eth(packet_parser_t *prs, eth = (const _odp_ethhdr_t *)*parseptr; /* Detect jumbo frames */ - if (odp_unlikely(frame_len > _ODP_ETH_LEN_MAX)) + if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX)) input_flags.jumbo = 1; /* Handle Ethernet broadcast/multicast addresses */ @@ -124,6 +125,16 @@ static inline uint16_t dpdk_parse_eth(packet_parser_t *prs, *parseptr += sizeof(_odp_vlanhdr_t); } + /* + * The packet was too short for what we parsed. We just give up + * entirely without trying to parse what fits in the packet. + */ + if (odp_unlikely(*offset > frame_len)) { + input_flags.all = 0; + input_flags.l2 = 1; + ethtype = 0; + } + error: prs->input_flags.all |= input_flags.all; @@ -151,6 +162,7 @@ static inline uint8_t dpdk_parse_ipv4(packet_parser_t *prs, if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN || ver != 4 || + sizeof(*ipv4) > frame_len - *offset || (l3_len > frame_len - *offset))) { prs->flags.ip_err = 1; return 0; @@ -218,8 +230,9 @@ static inline uint8_t dpdk_parse_ipv6(packet_parser_t *prs, uint32_t l4_packet_type = mbuf_packet_type & RTE_PTYPE_L4_MASK; /* Basic sanity checks on IPv6 header */ - if ((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || - l3_len > frame_len - *offset) { + if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || + sizeof(*ipv6) > frame_len - *offset || + l3_len > frame_len - *offset)) { prs->flags.ip_err = 1; return 0; } diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index 437977771..889a270ea 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -172,9 +172,16 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pkt_hdr = packet_hdr(new_pkt); } } else { - _odp_packet_parse_layer(pkt_hdr, - pktio_entry->s.config.parser.layer, - pktio_entry->s.in_chksums); + odp_packet_parse_param_t param; + + /* + * Use odp_packet_parse() which can handle segmented + * packets. + */ + param.proto = ODP_PROTO_ETH; + param.last_layer = pktio_entry->s.config.parser.layer; + param.chksums = pktio_entry->s.in_chksums; + odp_packet_parse(packet_handle(pkt_hdr), 0, ¶m); } packet_set_ts(pkt_hdr, ts); diff --git a/scripts/spelling.txt b/scripts/spelling.txt index 2e3ba91a5..69b5537a7 100644 --- a/scripts/spelling.txt +++ b/scripts/spelling.txt @@ -22,6 +22,7 @@ absolut||absolute absoulte||absolute acccess||access acceess||access +accelaration||acceleration acceleratoin||acceleration accelleration||acceleration accesing||accessing @@ -84,6 +85,7 @@ againt||against agaist||against aggreataon||aggregation aggreation||aggregation +ajust||adjust albumns||albums alegorical||allegorical algined||aligned @@ -161,10 +163,13 @@ asign||assign asser||assert assertation||assertion assertting||asserting +assgined||assigned assiged||assigned assigment||assignment assigments||assignments assistent||assistant +assocaited||associated +assocating||associating assocation||association associcated||associated assotiated||associated @@ -177,9 +182,11 @@ asynchnous||asynchronous asynchromous||asynchronous asymetric||asymmetric asymmeric||asymmetric +atleast||at least atomatically||automatically atomicly||atomically atempt||attempt +atrributes||attributes attachement||attachment attatch||attach attched||attached @@ -258,6 +265,7 @@ calucate||calculate calulate||calculate cancelation||cancellation cancle||cancel +canot||cannot capabilites||capabilities capabilties||capabilities capabilty||capability @@ -315,6 +323,7 @@ comminucation||communication commited||committed commiting||committing committ||commit +commnunication||communication commoditiy||commodity comsume||consume comsumer||consumer @@ -349,6 +358,7 @@ condtion||condition conected||connected conector||connector configration||configuration +configred||configured configuartion||configuration configuation||configuration configued||configured @@ -402,6 +412,7 @@ cunter||counter curently||currently cylic||cyclic dafault||default +deactive||deactivate deafult||default deamon||daemon debouce||debounce @@ -417,6 +428,7 @@ deffered||deferred defferred||deferred definate||definite definately||definitely +definiation||definition defintion||definition defintions||definitions defualt||default @@ -470,6 +482,7 @@ devided||divided deviece||device devision||division diable||disable +diabled||disabled dicline||decline dictionnary||dictionary didnt||didn't @@ -483,7 +496,10 @@ digial||digital dimention||dimension dimesions||dimensions diconnected||disconnected +disabed||disabled +disble||disable disgest||digest +disired||desired dispalying||displaying diplay||display directon||direction @@ -571,8 +587,9 @@ errror||error estbalishment||establishment etsablishment||establishment etsbalishment||establishment +evalute||evaluate +evalutes||evaluates evalution||evaluation -exeeds||exceeds excecutable||executable exceded||exceeded exceds||exceeds @@ -696,7 +713,9 @@ hardare||hardware harware||hardware havind||having heirarchically||hierarchically +heirarchy||hierarchy helpfull||helpful +hearbeat||heartbeat heterogenous||heterogeneous hexdecimal||hexadecimal hybernate||hibernate @@ -796,6 +815,7 @@ interanl||internal interchangable||interchangeable interferring||interfering interger||integer +intergrated||integrated intermittant||intermittent internel||internal interoprability||interoperability @@ -808,6 +828,7 @@ interrup||interrupt interrups||interrupts interruptted||interrupted interupted||interrupted +intiailized||initialized intial||initial intialisation||initialisation intialised||initialised @@ -974,6 +995,7 @@ notications||notifications notifcations||notifications notifed||notified notity||notify +nubmer||number numebr||number numner||number obtaion||obtain @@ -999,8 +1021,10 @@ ommiting||omitting ommitted||omitted onself||oneself ony||only +openning||opening operatione||operation opertaions||operations +opportunies||opportunities optionnal||optional optmizations||optimizations orientatied||orientated @@ -1013,6 +1037,8 @@ oustanding||outstanding overaall||overall overhread||overhead overlaping||overlapping +overflw||overflow +overlfow||overflow overide||override overrided||overridden overriden||overridden @@ -1091,11 +1117,15 @@ preemptable||preemptible prefered||preferred prefferably||preferably prefitler||prefilter +preform||perform premption||preemption prepaired||prepared +prepate||prepare preperation||preparation preprare||prepare pressre||pressure +presuambly||presumably +previosuly||previously primative||primitive princliple||principle priorty||priority @@ -1103,6 +1133,7 @@ privilaged||privileged privilage||privilege priviledge||privilege priviledges||privileges +privleges||privileges probaly||probably procceed||proceed proccesors||processors @@ -1147,6 +1178,7 @@ promixity||proximity psudo||pseudo psuedo||pseudo psychadelic||psychedelic +purgable||purgeable pwoer||power queing||queuing quering||querying @@ -1160,6 +1192,7 @@ receieve||receive recepient||recipient recevied||received receving||receiving +recievd||received recieved||received recieve||receive reciever||receiver @@ -1208,6 +1241,7 @@ reponse||response representaion||representation reqeust||request reqister||register +requed||requeued requestied||requested requiere||require requirment||requirement @@ -1265,6 +1299,7 @@ scarch||search schdule||schedule seach||search searchs||searches +secion||section secquence||sequence secund||second segement||segment @@ -1311,7 +1346,10 @@ singal||signal singed||signed sleeped||slept sliped||slipped +softwade||software softwares||software +soley||solely +souce||source speach||speech specfic||specific specfield||specified @@ -1320,7 +1358,9 @@ specifc||specific specifed||specified specificatin||specification specificaton||specification +specificed||specified specifing||specifying +specifiy||specify specifiying||specifying speficied||specified speicify||specify @@ -1436,6 +1476,7 @@ timout||timeout tmis||this toogle||toggle torerable||tolerable +traget||target traking||tracking tramsmitted||transmitted tramsmit||transmit @@ -1484,6 +1525,7 @@ unintialized||uninitialized unitialized||uninitialized unkmown||unknown unknonw||unknown +unknouwn||unknown unknow||unknown unkown||unknown unamed||unnamed @@ -1558,9 +1600,15 @@ wiil||will wirte||write withing||within wnat||want +wont||won't workarould||workaround writeing||writing writting||writing wtih||with zombe||zombie zomebie||zombie + +# +# ODP additions +# +odp_cache_aligned||ODP_ALIGNED_CACHE diff --git a/test/common/odp_cunit_common.c b/test/common/odp_cunit_common.c index f5c437344..c6e03dab2 100644 --- a/test/common/odp_cunit_common.c +++ b/test/common/odp_cunit_common.c @@ -54,8 +54,8 @@ int odp_cunit_thread_create(int func_ptr(void *), pthrd_arg *arg) int num = arg->numthrds; odph_thread_param_t thr_param; - memset(&thr_common, 0, sizeof(thr_common)); - memset(&thr_param, 0, sizeof(thr_param)); + odph_thread_common_param_init(&thr_common); + odph_thread_param_init(&thr_param); thr_param.start = func_ptr; thr_param.arg = arg; diff --git a/test/performance/odp_atomic_perf.c b/test/performance/odp_atomic_perf.c index b56135c35..ee760babf 100644 --- a/test/performance/odp_atomic_perf.c +++ b/test/performance/odp_atomic_perf.c @@ -1076,11 +1076,10 @@ static int start_workers(test_global_t *global, odp_instance_t instance, int num_cpu = test_options->num_cpu; odph_thread_param_t thr_param[num_cpu]; - memset(¶m, 0, sizeof(odph_thread_common_param_t)); + odph_thread_common_param_init(¶m); param.instance = instance; param.cpumask = &global->cpumask; - memset(thr_param, 0, sizeof(thr_param)); for (i = 0; i < num_cpu; i++) { test_thread_ctx_t *thread_ctx = &global->thread_ctx[i]; @@ -1089,6 +1088,7 @@ static int start_workers(test_global_t *global, odp_instance_t instance, thread_ctx->func = func; thread_ctx->type = type; + odph_thread_param_init(&thr_param[i]); thr_param[i].thr_type = ODP_THREAD_WORKER; thr_param[i].start = run_test; thr_param[i].arg = thread_ctx; diff --git a/test/performance/odp_bench_packet.c b/test/performance/odp_bench_packet.c index e80e823f6..0354ef9b8 100644 --- a/test/performance/odp_bench_packet.c +++ b/test/performance/odp_bench_packet.c @@ -1739,7 +1739,9 @@ bench_info_t test_suite[] = { int main(int argc, char *argv[]) { odph_helper_options_t helper_options; - odph_odpthread_t worker_thread; + odph_thread_t worker_thread; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; int cpu; odp_shm_t shm; odp_cpumask_t cpumask; @@ -1864,7 +1866,7 @@ int main(int argc, char *argv[]) odp_pool_print(gbl_args->pool); - memset(&worker_thread, 0, sizeof(odph_odpthread_t)); + memset(&worker_thread, 0, sizeof(odph_thread_t)); signal(SIGINT, sig_handler); @@ -1872,20 +1874,23 @@ int main(int argc, char *argv[]) cpu = odp_cpumask_first(&cpumask); odp_cpumask_t thd_mask; - odph_odpthread_params_t thr_params; - - memset(&thr_params, 0, sizeof(thr_params)); - thr_params.start = run_benchmarks; - thr_params.arg = gbl_args; - thr_params.thr_type = ODP_THREAD_WORKER; - thr_params.instance = instance; odp_cpumask_zero(&thd_mask); odp_cpumask_set(&thd_mask, cpu); - odph_odpthreads_create(&worker_thread, &thd_mask, - &thr_params); - odph_odpthreads_join(&worker_thread); + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &thd_mask; + thr_common.share_param = 1; + + odph_thread_param_init(&thr_param); + thr_param.start = run_benchmarks; + thr_param.arg = gbl_args; + thr_param.thr_type = ODP_THREAD_WORKER; + + odph_thread_create(&worker_thread, &thr_common, &thr_param, 1); + + odph_thread_join(&worker_thread, 1); ret = gbl_args->bench_failed; diff --git a/test/performance/odp_cpu_bench.c b/test/performance/odp_cpu_bench.c index a4999ae27..e0ac82846 100644 --- a/test/performance/odp_cpu_bench.c +++ b/test/performance/odp_cpu_bench.c @@ -521,7 +521,9 @@ int main(int argc, char *argv[]) { stats_t *stats[MAX_WORKERS]; odph_helper_options_t helper_options; - odph_odpthread_t thread_tbl[MAX_WORKERS]; + odph_thread_t thread_tbl[MAX_WORKERS]; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param[MAX_WORKERS]; odp_cpumask_t cpumask; odp_pool_capability_t pool_capa; odp_pool_t pool; @@ -540,7 +542,6 @@ int main(int argc, char *argv[]) uint32_t init_val; unsigned int num_workers; unsigned int i, j; - int cpu; int ret = 0; /* Let helper collect its own arguments (e.g. --odph_proc) */ @@ -743,7 +744,6 @@ int main(int argc, char *argv[]) } } - memset(thread_tbl, 0, sizeof(thread_tbl)); odp_barrier_init(&gbl_args->init_barrier, num_workers + 1); odp_barrier_init(&gbl_args->term_barrier, num_workers + 1); @@ -762,34 +762,28 @@ int main(int argc, char *argv[]) } /* Create worker threads */ - cpu = odp_cpumask_first(&cpumask); - for (i = 0; i < num_workers; i++) { - odp_cpumask_t thd_mask; - odph_odpthread_params_t thr_params; + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &cpumask; + for (i = 0; i < num_workers; i++) { gbl_args->thread[i].idx = i; - - memset(&thr_params, 0, sizeof(thr_params)); - thr_params.start = run_thread; - thr_params.arg = &gbl_args->thread[i]; - thr_params.thr_type = ODP_THREAD_WORKER; - thr_params.instance = instance; - stats[i] = &gbl_args->thread[i].stats; - odp_cpumask_zero(&thd_mask); - odp_cpumask_set(&thd_mask, cpu); - odph_odpthreads_create(&thread_tbl[i], &thd_mask, - &thr_params); - cpu = odp_cpumask_next(&cpumask, cpu); + odph_thread_param_init(&thr_param[i]); + thr_param[i].start = run_thread; + thr_param[i].arg = &gbl_args->thread[i]; + thr_param[i].thr_type = ODP_THREAD_WORKER; } + memset(thread_tbl, 0, sizeof(thread_tbl)); + odph_thread_create(thread_tbl, &thr_common, thr_param, num_workers); + ret = print_stats(num_workers, stats, gbl_args->appl.time, gbl_args->appl.accuracy); /* Master thread waits for other threads to exit */ - for (i = 0; i < num_workers; ++i) - odph_odpthreads_join(&thread_tbl[i]); + odph_thread_join(thread_tbl, num_workers); for (i = 0; i < num_groups; i++) { for (j = 0; j < QUEUES_PER_GROUP; j++) { diff --git a/test/performance/odp_crypto.c b/test/performance/odp_crypto.c index 36324622a..4f81dab17 100644 --- a/test/performance/odp_crypto.c +++ b/test/performance/odp_crypto.c @@ -1032,7 +1032,9 @@ int main(int argc, char *argv[]) char cpumaskstr[ODP_CPUMASK_STR_SIZE]; int num_workers = 1; odph_helper_options_t helper_options; - odph_odpthread_t thr[num_workers]; + odph_thread_t thread_tbl[num_workers]; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; odp_instance_t instance; odp_init_t init_param; odp_pool_capability_t pool_capa; @@ -1146,24 +1148,26 @@ int main(int argc, char *argv[]) printf("Run in sync mode\n"); } - memset(thr, 0, sizeof(thr)); - test_run_arg.crypto_args = cargs; test_run_arg.crypto_alg_config = cargs.alg_config; test_run_arg.crypto_capa = crypto_capa; if (cargs.alg_config) { - odph_odpthread_params_t thr_params; - - memset(&thr_params, 0, sizeof(thr_params)); - thr_params.start = run_thr_func; - thr_params.arg = &test_run_arg; - thr_params.thr_type = ODP_THREAD_WORKER; - thr_params.instance = instance; + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &cpumask; + thr_common.share_param = 1; if (cargs.schedule) { - odph_odpthreads_create(&thr[0], &cpumask, &thr_params); - odph_odpthreads_join(&thr[0]); + odph_thread_param_init(&thr_param); + thr_param.start = run_thr_func; + thr_param.arg = &test_run_arg; + thr_param.thr_type = ODP_THREAD_WORKER; + + memset(thread_tbl, 0, sizeof(thread_tbl)); + odph_thread_create(thread_tbl, &thr_common, &thr_param, num_workers); + + odph_thread_join(thread_tbl, num_workers); } else { run_measure_one_config(&test_run_arg); } diff --git a/test/performance/odp_ipsec.c b/test/performance/odp_ipsec.c index 05a22ff0d..04788995e 100644 --- a/test/performance/odp_ipsec.c +++ b/test/performance/odp_ipsec.c @@ -1027,9 +1027,12 @@ int main(int argc, char *argv[]) char cpumaskstr[ODP_CPUMASK_STR_SIZE]; int num_workers = 1; odph_helper_options_t helper_options; - odph_odpthread_t thr[num_workers]; + odph_thread_t thread_tbl[num_workers]; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; odp_instance_t instance; odp_init_t init_param; + odp_ipsec_capability_t ipsec_capa; odp_pool_capability_t capa; odp_ipsec_config_t config; uint32_t max_seg_len; @@ -1092,6 +1095,21 @@ int main(int argc, char *argv[]) } odp_pool_print(pool); + if (odp_ipsec_capability(&ipsec_capa) < 0) { + app_err("IPSEC capability call failed.\n"); + exit(EXIT_FAILURE); + } + + if (cargs.schedule && !ipsec_capa.queue_type_sched) { + app_err("Scheduled type destination queue not supported.\n"); + exit(EXIT_FAILURE); + } + + if (cargs.poll && !ipsec_capa.queue_type_plain) { + app_err("Plain type destination queue not supported.\n"); + exit(EXIT_FAILURE); + } + odp_ipsec_config_init(&config); config.max_num_sa = 2; config.inbound.chksums.all_chksum = 0; @@ -1148,20 +1166,22 @@ int main(int argc, char *argv[]) printf("Run in sync mode\n"); } - memset(thr, 0, sizeof(thr)); - if (cargs.alg_config) { - odph_odpthread_params_t thr_param; - - memset(&thr_param, 0, sizeof(thr_param)); - thr_param.start = run_thr_func; - thr_param.arg = &thr_arg; - thr_param.thr_type = ODP_THREAD_WORKER; - thr_param.instance = instance; + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &cpumask; + thr_common.share_param = 1; if (cargs.schedule) { - odph_odpthreads_create(&thr[0], &cpumask, &thr_param); - odph_odpthreads_join(&thr[0]); + odph_thread_param_init(&thr_param); + thr_param.start = run_thr_func; + thr_param.arg = &thr_arg; + thr_param.thr_type = ODP_THREAD_WORKER; + + memset(thread_tbl, 0, sizeof(thread_tbl)); + odph_thread_create(thread_tbl, &thr_common, &thr_param, num_workers); + + odph_thread_join(thread_tbl, num_workers); } else { run_measure_one_config(&cargs, cargs.alg_config); } diff --git a/test/performance/odp_l2fwd.c b/test/performance/odp_l2fwd.c index 3da08661c..6aa98ce96 100644 --- a/test/performance/odp_l2fwd.c +++ b/test/performance/odp_l2fwd.c @@ -1,6 +1,6 @@ /* Copyright (c) 2014-2018, Linaro Limited * Copyright (c) 2019-2021, Nokia - * Copyright (c) 2020, Marvell + * Copyright (c) 2020-2021, Marvell * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -108,6 +108,7 @@ typedef struct { uint32_t packet_len; /* Maximum packet length supported */ uint32_t seg_len; /* Pool segment length */ int promisc_mode; /* Promiscuous mode enabled */ + int flow_aware; /* Flow aware scheduling enabled */ int mtu; /* Interface MTU */ } appl_args_t; @@ -1547,6 +1548,7 @@ static void usage(char *progname) " -l, --packet_len <len> Maximum length of packets supported (default %d).\n" " -L, --seg_len <len> Packet pool segment length\n" " (default equal to packet length).\n" + " -f, --flow_aware Enable flow aware scheduling.\n" " -v, --verbose Verbose output.\n" " -h, --help Display help and exit.\n\n" "\n", NO_PATH(progname), NO_PATH(progname), MAX_PKTIOS, DEFAULT_VEC_SIZE, @@ -1594,12 +1596,13 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) {"promisc_mode", no_argument, NULL, 'P'}, {"packet_len", required_argument, NULL, 'l'}, {"seg_len", required_argument, NULL, 'L'}, + {"flow_aware", no_argument, NULL, 'f'}, {"verbose", no_argument, NULL, 'v'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; - static const char *shortopts = "+c:t:a:i:m:o:r:d:s:e:k:g:b:p:y:n:l:L:w:x:z:M:uPvh"; + static const char *shortopts = "+c:t:a:i:m:o:r:d:s:e:k:g:b:p:y:n:l:L:w:x:z:M:uPfvh"; appl_args->time = 0; /* loop forever if time to run is 0 */ appl_args->accuracy = 1; /* get and print pps stats second */ @@ -1622,6 +1625,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) appl_args->num_vec = 0; appl_args->vec_size = 0; appl_args->vec_tmo_ns = 0; + appl_args->flow_aware = 0; while (1) { opt = getopt_long(argc, argv, shortopts, longopts, &long_index); @@ -1786,6 +1790,9 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) case 'z': appl_args->vec_tmo_ns = atoi(optarg); break; + case 'f': + appl_args->flow_aware = 1; + break; case 'v': appl_args->verbose = 1; break; @@ -1869,6 +1876,8 @@ static void print_info(appl_args_t *appl_args) printf("interface default\n"); printf("Promisc mode: %s\n", appl_args->promisc_mode ? "enabled" : "disabled"); + printf("Flow aware: %s\n", appl_args->flow_aware ? + "yes" : "no"); printf("Burst size: %i\n", appl_args->burst_rx); printf("Number of pools: %i\n", appl_args->pool_per_if ? appl_args->if_count : 1); @@ -1988,6 +1997,8 @@ int main(int argc, char *argv[]) odp_pool_t pool, vec_pool; odp_init_t init; odp_pool_capability_t pool_capa; + odp_schedule_config_t sched_config; + odp_schedule_capability_t sched_capa; uint32_t pkt_len, num_pkt, seg_len; /* Let helper collect its own arguments (e.g. --odph_proc) */ @@ -2210,7 +2221,23 @@ int main(int argc, char *argv[]) bind_workers(); - odp_schedule_config(NULL); + odp_schedule_config_init(&sched_config); + + if (odp_schedule_capability(&sched_capa)) { + ODPH_ERR("Error: schedule capability failed\n"); + exit(EXIT_FAILURE); + } + + if (gbl_args->appl.flow_aware) { + if (sched_capa.max_flow_id) { + sched_config.max_flow_id = sched_capa.max_flow_id; + } else { + ODPH_ERR("Error: flow aware mode not supported\n"); + exit(EXIT_FAILURE); + } + } + + odp_schedule_config(&sched_config); /* Default */ if (num_groups == 0) { @@ -2296,8 +2323,7 @@ int main(int argc, char *argv[]) run_worker_sched_mode_vector : run_worker_sched_mode; /* Create worker threads */ - memset(thr_param, 0, sizeof(thr_param)); - memset(&thr_common, 0, sizeof(thr_common)); + odph_thread_common_param_init(&thr_common); thr_common.instance = instance; thr_common.cpumask = &cpumask; @@ -2306,6 +2332,7 @@ int main(int argc, char *argv[]) thr_common.sync = 1; for (i = 0; i < num_workers; ++i) { + odph_thread_param_init(&thr_param[i]); thr_param[i].start = thr_run_func; thr_param[i].arg = &gbl_args->thread_args[i]; thr_param[i].thr_type = ODP_THREAD_WORKER; diff --git a/test/performance/odp_mem_perf.c b/test/performance/odp_mem_perf.c index 1348eaaa2..56a3cdf9a 100644 --- a/test/performance/odp_mem_perf.c +++ b/test/performance/odp_mem_perf.c @@ -311,11 +311,10 @@ static int start_workers(test_global_t *global, odp_instance_t instance) int num_cpu = test_options->num_cpu; odph_thread_param_t thr_param[num_cpu]; - memset(¶m, 0, sizeof(odph_thread_common_param_t)); + odph_thread_common_param_init(¶m); param.instance = instance; param.cpumask = &global->cpumask; - memset(thr_param, 0, sizeof(thr_param)); for (i = 0; i < num_cpu; i++) { test_thread_ctx_t *thread_ctx = &global->thread_ctx[i]; @@ -324,6 +323,7 @@ static int start_workers(test_global_t *global, odp_instance_t instance) if (global->test_options.private) thread_ctx->shm_addr = global->shm_addr[i]; + odph_thread_param_init(&thr_param[i]); thr_param[i].thr_type = ODP_THREAD_WORKER; thr_param[i].start = run_test; thr_param[i].arg = thread_ctx; diff --git a/test/performance/odp_packet_gen.c b/test/performance/odp_packet_gen.c index f3919ebac..02704b195 100644 --- a/test/performance/odp_packet_gen.c +++ b/test/performance/odp_packet_gen.c @@ -1486,14 +1486,14 @@ static int start_workers(test_global_t *global, odp_instance_t instance) odph_thread_param_t thr_param[num_cpu]; memset(global->thread_tbl, 0, sizeof(global->thread_tbl)); - memset(thr_param, 0, sizeof(thr_param)); - memset(&thr_common, 0, sizeof(thr_common)); + odph_thread_common_param_init(&thr_common); thr_common.instance = instance; thr_common.cpumask = &global->cpumask; /* Receive threads */ for (i = 0; i < num_rx; i++) { + odph_thread_param_init(&thr_param[i]); thr_param[i].start = rx_thread; thr_param[i].arg = &global->thread_arg[i]; thr_param[i].thr_type = ODP_THREAD_WORKER; @@ -1513,6 +1513,7 @@ static int start_workers(test_global_t *global, odp_instance_t instance) global->thread_arg[i].pktout[j] = pktout; } + odph_thread_param_init(&thr_param[i]); thr_param[i].start = tx_thread; thr_param[i].arg = &global->thread_arg[i]; thr_param[i].thr_type = ODP_THREAD_WORKER; diff --git a/test/performance/odp_pktio_ordered.c b/test/performance/odp_pktio_ordered.c index d5ffcc8ab..e35386d52 100644 --- a/test/performance/odp_pktio_ordered.c +++ b/test/performance/odp_pktio_ordered.c @@ -1060,10 +1060,11 @@ int main(int argc, char *argv[]) odp_pool_capability_t pool_capa; odph_ethaddr_t new_addr; odph_helper_options_t helper_options; - odph_odpthread_t thread_tbl[MAX_WORKERS]; + odph_thread_t thread_tbl[MAX_WORKERS]; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param[MAX_WORKERS]; stats_t *stats; char cpumaskstr[ODP_CPUMASK_STR_SIZE]; - int cpu; int i, j; int if_count; int ret; @@ -1281,26 +1282,21 @@ int main(int argc, char *argv[]) odp_barrier_init(&gbl_args->barrier, num_workers + 1); /* Create worker threads */ - cpu = odp_cpumask_first(&cpumask); - for (i = 0; i < num_workers; ++i) { - odp_cpumask_t thd_mask; - odph_odpthread_params_t thr_params; - - memset(&thr_params, 0, sizeof(thr_params)); - thr_params.start = run_worker; - thr_params.arg = &gbl_args->thread[i]; - thr_params.thr_type = ODP_THREAD_WORKER; - thr_params.instance = instance; + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &cpumask; + for (i = 0; i < num_workers; ++i) { gbl_args->thread[i].stats = &stats[i]; - odp_cpumask_zero(&thd_mask); - odp_cpumask_set(&thd_mask, cpu); - odph_odpthreads_create(&thread_tbl[i], &thd_mask, - &thr_params); - cpu = odp_cpumask_next(&cpumask, cpu); + odph_thread_param_init(&thr_param[i]); + thr_param[i].start = run_worker; + thr_param[i].arg = &gbl_args->thread[i]; + thr_param[i].thr_type = ODP_THREAD_WORKER; } + odph_thread_create(thread_tbl, &thr_common, thr_param, num_workers); + /* Start packet receive and transmit */ for (i = 0; i < if_count; ++i) { odp_pktio_t pktio; @@ -1324,8 +1320,7 @@ int main(int argc, char *argv[]) odp_atomic_store_u32(&gbl_args->exit_threads, 1); /* Master thread waits for other threads to exit */ - for (i = 0; i < num_workers; ++i) - odph_odpthreads_join(&thread_tbl[i]); + odph_thread_join(thread_tbl, num_workers); for (i = 0; i < if_count; i++) { odp_pktio_close(gbl_args->pktios[i].pktio); diff --git a/test/performance/odp_pktio_perf.c b/test/performance/odp_pktio_perf.c index 3d70d7d2b..593465f4f 100644 --- a/test/performance/odp_pktio_perf.c +++ b/test/performance/odp_pktio_perf.c @@ -603,45 +603,61 @@ static int run_test_single(odp_cpumask_t *thd_mask_tx, odp_cpumask_t *thd_mask_rx, test_status_t *status) { - odph_odpthread_t thd_tbl[MAX_WORKERS]; + odph_thread_t thread_tbl[MAX_WORKERS]; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; thread_args_t args_tx, args_rx; uint64_t expected_tx_cnt; int num_tx_workers, num_rx_workers; - odph_odpthread_params_t thr_params; - - memset(&thr_params, 0, sizeof(thr_params)); - thr_params.thr_type = ODP_THREAD_WORKER; - thr_params.instance = gbl_args->instance; odp_atomic_store_u32(&gbl_args->shutdown, 0); - memset(thd_tbl, 0, sizeof(thd_tbl)); + memset(thread_tbl, 0, sizeof(thread_tbl)); memset(gbl_args->rx_stats, 0, gbl_args->rx_stats_size); memset(gbl_args->tx_stats, 0, gbl_args->tx_stats_size); expected_tx_cnt = status->pps_curr * gbl_args->args.duration; /* start receiver threads first */ - thr_params.start = run_thread_rx; - thr_params.arg = &args_rx; + + num_rx_workers = odp_cpumask_count(thd_mask_rx); args_rx.batch_len = gbl_args->args.rx_batch_len; - odph_odpthreads_create(&thd_tbl[0], thd_mask_rx, &thr_params); + + odph_thread_common_param_init(&thr_common); + thr_common.instance = gbl_args->instance; + thr_common.cpumask = thd_mask_rx; + thr_common.share_param = 1; + + odph_thread_param_init(&thr_param); + thr_param.start = run_thread_rx; + thr_param.arg = &args_rx; + thr_param.thr_type = ODP_THREAD_WORKER; + + odph_thread_create(thread_tbl, &thr_common, &thr_param, num_rx_workers); odp_barrier_wait(&gbl_args->rx_barrier); - num_rx_workers = odp_cpumask_count(thd_mask_rx); /* then start transmitters */ - thr_params.start = run_thread_tx; - thr_params.arg = &args_tx; + num_tx_workers = odp_cpumask_count(thd_mask_tx); args_tx.pps = status->pps_curr / num_tx_workers; args_tx.duration = gbl_args->args.duration; args_tx.batch_len = gbl_args->args.tx_batch_len; - odph_odpthreads_create(&thd_tbl[num_rx_workers], thd_mask_tx, - &thr_params); + + odph_thread_common_param_init(&thr_common); + thr_common.instance = gbl_args->instance; + thr_common.cpumask = thd_mask_tx; + thr_common.share_param = 1; + + odph_thread_param_init(&thr_param); + thr_param.start = run_thread_tx; + thr_param.arg = &args_tx; + thr_param.thr_type = ODP_THREAD_WORKER; + + odph_thread_create(&thread_tbl[num_rx_workers], &thr_common, &thr_param, num_tx_workers); odp_barrier_wait(&gbl_args->tx_barrier); /* wait for transmitter threads to terminate */ - odph_odpthreads_join(&thd_tbl[num_rx_workers]); + odph_thread_join(&thread_tbl[num_rx_workers], num_tx_workers); /* delay to allow transmitted packets to reach the receivers */ odp_time_wait_ns(SHUTDOWN_DELAY_NS); @@ -650,7 +666,7 @@ static int run_test_single(odp_cpumask_t *thd_mask_tx, odp_atomic_store_u32(&gbl_args->shutdown, 1); /* wait for receivers */ - odph_odpthreads_join(&thd_tbl[0]); + odph_thread_join(thread_tbl, num_rx_workers); if (!status->warmup) return process_results(expected_tx_cnt, status); diff --git a/test/performance/odp_pool_perf.c b/test/performance/odp_pool_perf.c index ee97af519..957b1de00 100644 --- a/test/performance/odp_pool_perf.c +++ b/test/performance/odp_pool_perf.c @@ -43,7 +43,7 @@ typedef struct test_global_t { odp_barrier_t barrier; odp_pool_t pool; odp_cpumask_t cpumask; - odph_odpthread_t thread_tbl[ODP_THREAD_COUNT_MAX]; + odph_thread_t thread_tbl[ODP_THREAD_COUNT_MAX]; test_stat_t stat[ODP_THREAD_COUNT_MAX]; } test_global_t; @@ -445,23 +445,28 @@ static int test_packet_pool(void *arg) static int start_workers(test_global_t *global, odp_instance_t instance) { - odph_odpthread_params_t thr_params; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; test_options_t *test_options = &global->test_options; int num_cpu = test_options->num_cpu; int packet_pool = test_options->pool_type; - memset(&thr_params, 0, sizeof(thr_params)); - thr_params.thr_type = ODP_THREAD_WORKER; - thr_params.instance = instance; - thr_params.arg = global; + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &global->cpumask; + thr_common.share_param = 1; + + odph_thread_param_init(&thr_param); + thr_param.arg = global; + thr_param.thr_type = ODP_THREAD_WORKER; if (packet_pool) - thr_params.start = test_packet_pool; + thr_param.start = test_packet_pool; else - thr_params.start = test_buffer_pool; + thr_param.start = test_buffer_pool; - if (odph_odpthreads_create(global->thread_tbl, &global->cpumask, - &thr_params) != num_cpu) + if (odph_thread_create(global->thread_tbl, &thr_common, &thr_param, + num_cpu) != num_cpu) return -1; return 0; @@ -608,7 +613,7 @@ int main(int argc, char **argv) start_workers(global, instance); /* Wait workers to exit */ - odph_odpthreads_join(global->thread_tbl); + odph_thread_join(global->thread_tbl, global->test_options.num_cpu); print_stat(global); diff --git a/test/performance/odp_queue_perf.c b/test/performance/odp_queue_perf.c index 33284d312..320f2f35a 100644 --- a/test/performance/odp_queue_perf.c +++ b/test/performance/odp_queue_perf.c @@ -44,7 +44,7 @@ typedef struct test_global_t { odp_shm_t shm; odp_pool_t pool; odp_queue_t queue[MAX_QUEUES]; - odph_odpthread_t thread_tbl[ODP_THREAD_COUNT_MAX]; + odph_thread_t thread_tbl[ODP_THREAD_COUNT_MAX]; test_stat_t stat[ODP_THREAD_COUNT_MAX]; } test_global_t; @@ -423,18 +423,13 @@ error: static int start_workers(test_global_t *global) { - odph_odpthread_params_t thr_params; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; odp_cpumask_t cpumask; int ret; test_options_t *test_options = &global->options; int num_cpu = test_options->num_cpu; - memset(&thr_params, 0, sizeof(thr_params)); - thr_params.thr_type = ODP_THREAD_WORKER; - thr_params.instance = global->instance; - thr_params.start = run_test; - thr_params.arg = global; - ret = odp_cpumask_default_worker(&cpumask, num_cpu); if (num_cpu && ret != num_cpu) { @@ -452,8 +447,18 @@ static int start_workers(test_global_t *global) odp_barrier_init(&global->barrier, num_cpu); - if (odph_odpthreads_create(global->thread_tbl, &cpumask, &thr_params) - != num_cpu) + odph_thread_common_param_init(&thr_common); + thr_common.instance = global->instance; + thr_common.cpumask = &cpumask; + thr_common.share_param = 1; + + odph_thread_param_init(&thr_param); + thr_param.start = run_test; + thr_param.arg = global; + thr_param.thr_type = ODP_THREAD_WORKER; + + if (odph_thread_create(global->thread_tbl, &thr_common, &thr_param, + num_cpu) != num_cpu) return -1; return 0; @@ -596,7 +601,7 @@ int main(int argc, char **argv) } /* Wait workers to exit */ - odph_odpthreads_join(global->thread_tbl); + odph_thread_join(global->thread_tbl, global->options.num_cpu); print_stat(global); diff --git a/test/performance/odp_sched_latency.c b/test/performance/odp_sched_latency.c index c6b659aac..2910dcdbc 100644 --- a/test/performance/odp_sched_latency.c +++ b/test/performance/odp_sched_latency.c @@ -705,8 +705,9 @@ int main(int argc, char *argv[]) odp_instance_t instance; odp_init_t init_param; odph_helper_options_t helper_options; - odph_odpthread_t *thread_tbl; - odph_odpthread_params_t thr_params; + odph_thread_t *thread_tbl; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; odp_cpumask_t cpumask; odp_pool_t pool; odp_pool_capability_t pool_capa; @@ -766,7 +767,7 @@ int main(int argc, char *argv[]) printf(" First CPU: %i\n", odp_cpumask_first(&cpumask)); printf(" CPU mask: %s\n", cpumaskstr); - thread_tbl = calloc(sizeof(odph_odpthread_t), num_workers); + thread_tbl = calloc(sizeof(odph_thread_t), num_workers); if (!thread_tbl) { ODPH_ERR("no memory for thread_tbl\n"); return -1; @@ -858,15 +859,20 @@ int main(int argc, char *argv[]) odp_barrier_init(&globals->barrier, num_workers); /* Create and launch worker threads */ - memset(&thr_params, 0, sizeof(thr_params)); - thr_params.thr_type = ODP_THREAD_WORKER; - thr_params.instance = instance; - thr_params.start = run_thread; - thr_params.arg = NULL; - odph_odpthreads_create(thread_tbl, &cpumask, &thr_params); + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &cpumask; + thr_common.share_param = 1; + + odph_thread_param_init(&thr_param); + thr_param.start = run_thread; + thr_param.arg = NULL; + thr_param.thr_type = ODP_THREAD_WORKER; + + odph_thread_create(thread_tbl, &thr_common, &thr_param, num_workers); /* Wait for worker threads to terminate */ - odph_odpthreads_join(thread_tbl); + odph_thread_join(thread_tbl, num_workers); free(thread_tbl); printf("ODP scheduling latency test complete\n\n"); diff --git a/test/performance/odp_sched_perf.c b/test/performance/odp_sched_perf.c index da818931c..f9b4ce548 100644 --- a/test/performance/odp_sched_perf.c +++ b/test/performance/odp_sched_perf.c @@ -979,10 +979,12 @@ static int test_sched(void *arg) odp_event_t event; uint64_t sched_wait = odp_schedule_wait_time(200 * ODP_TIME_MSEC_IN_NS); - /* Print schedule status at the end of the test, before any queues + /* Print queue and scheduler status at the end of the test, before any queues * are emptied or destroyed. */ - if (test_options->verbose) + if (test_options->verbose) { + odp_queue_print_all(); odp_schedule_print(); + } while ((event = odp_schedule(NULL, sched_wait)) != ODP_EVENT_INVALID) odp_event_free(event); @@ -1025,13 +1027,13 @@ static int start_workers(test_global_t *global, odp_instance_t instance) odp_atomic_init_u32(&global->num_worker, num_cpu); memset(global->thread_tbl, 0, sizeof(global->thread_tbl)); - memset(thr_param, 0, sizeof(thr_param)); - memset(&thr_common, 0, sizeof(thr_common)); + odph_thread_common_param_init(&thr_common); thr_common.instance = instance; thr_common.cpumask = &global->cpumask; for (i = 0; i < num_cpu; i++) { + odph_thread_param_init(&thr_param[i]); thr_param[i].start = test_sched; thr_param[i].arg = &global->thread_arg[i]; thr_param[i].thr_type = ODP_THREAD_WORKER; diff --git a/test/performance/odp_sched_pktio.c b/test/performance/odp_sched_pktio.c index cbdbdf4aa..589b58d97 100644 --- a/test/performance/odp_sched_pktio.c +++ b/test/performance/odp_sched_pktio.c @@ -1396,45 +1396,50 @@ static void destroy_timers(test_global_t *test_global) odp_timer_pool_destroy(timer_pool); } -static void start_workers(odph_odpthread_t thread[], +static void start_workers(odph_thread_t thread[], test_global_t *test_global) { int i; odp_cpumask_t cpumask; - odph_odpthread_params_t param; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param[MAX_WORKERS]; int num = test_global->opt.num_worker; - memset(¶m, 0, sizeof(odph_odpthread_params_t)); + odp_cpumask_zero(&cpumask); - if (test_global->opt.timeout_us) - param.start = worker_thread_timers; - else if (test_global->opt.pipe_stages) - param.start = worker_thread_pipeline; - else - param.start = worker_thread_direct; - - param.thr_type = ODP_THREAD_WORKER; - param.instance = test_global->instance; - - memset(thread, 0, num * sizeof(odph_odpthread_t)); + odph_thread_common_param_init(&thr_common); + thr_common.instance = test_global->instance; + thr_common.cpumask = &cpumask; for (i = 0; i < num; i++) { - odp_cpumask_zero(&cpumask); odp_cpumask_set(&cpumask, test_global->worker_cpu[i]); test_global->worker_arg[i].worker_id = i; test_global->worker_arg[i].test_global_ptr = test_global; - param.arg = &test_global->worker_arg[i]; - odph_odpthreads_create(&thread[i], &cpumask, ¶m); + odph_thread_param_init(&thr_param[i]); + + if (!i) { + if (test_global->opt.timeout_us) + thr_param[0].start = worker_thread_timers; + else if (test_global->opt.pipe_stages) + thr_param[0].start = worker_thread_pipeline; + else + thr_param[0].start = worker_thread_direct; + } else { + thr_param[i].start = thr_param[0].start; + } + + thr_param[i].arg = &test_global->worker_arg[i]; + thr_param[i].thr_type = ODP_THREAD_WORKER; } + + memset(thread, 0, num * sizeof(odph_thread_t)); + odph_thread_create(thread, &thr_common, thr_param, num); } -static void wait_workers(odph_odpthread_t thread[], test_global_t *test_global) +static void wait_workers(odph_thread_t thread[], test_global_t *test_global) { - int i; - - for (i = 0; i < test_global->opt.num_worker; ++i) - odph_odpthreads_join(&thread[i]); + odph_thread_join(thread, test_global->opt.num_worker); } int main(int argc, char *argv[]) @@ -1444,7 +1449,7 @@ int main(int argc, char *argv[]) odp_shm_t shm; odp_time_t t1 = ODP_TIME_NULL, t2 = ODP_TIME_NULL; odph_helper_options_t helper_options; - odph_odpthread_t thread[MAX_WORKERS]; + odph_thread_t thread[MAX_WORKERS]; test_options_t test_options; int ret = 0; diff --git a/test/performance/odp_scheduling.c b/test/performance/odp_scheduling.c index 76d767220..d52ad816a 100644 --- a/test/performance/odp_scheduling.c +++ b/test/performance/odp_scheduling.c @@ -997,17 +997,17 @@ int main(int argc, char *argv[]) globals->first_thr = -1; /* Create and launch worker threads */ - memset(&thr_common, 0, sizeof(thr_common)); - memset(&thr_param, 0, sizeof(thr_param)); - - thr_param.thr_type = ODP_THREAD_WORKER; - thr_param.start = run_thread; - thr_param.arg = NULL; + odph_thread_common_param_init(&thr_common); thr_common.instance = instance; thr_common.cpumask = &cpumask; thr_common.share_param = 1; + odph_thread_param_init(&thr_param); + thr_param.thr_type = ODP_THREAD_WORKER; + thr_param.start = run_thread; + thr_param.arg = NULL; + odph_thread_create(thread_tbl, &thr_common, &thr_param, num_workers); /* Wait for worker threads to terminate */ diff --git a/test/performance/odp_timer_perf.c b/test/performance/odp_timer_perf.c index 6989d0d4c..a45081643 100644 --- a/test/performance/odp_timer_perf.c +++ b/test/performance/odp_timer_perf.c @@ -795,13 +795,14 @@ static int start_workers(test_global_t *global, odp_instance_t instance) odph_thread_param_t thr_param[num_cpu]; memset(global->thread_tbl, 0, sizeof(global->thread_tbl)); - memset(thr_param, 0, sizeof(thr_param)); - memset(&thr_common, 0, sizeof(thr_common)); + odph_thread_common_param_init(&thr_common); thr_common.instance = instance; thr_common.cpumask = &global->cpumask; for (i = 0; i < num_cpu; i++) { + odph_thread_param_init(&thr_param[i]); + if (test_options->mode == MODE_SCHED_OVERH) thr_param[i].start = sched_mode_worker; else diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c index 7d6661be2..06ece9f50 100644 --- a/test/validation/api/crypto/odp_crypto_test_inp.c +++ b/test/validation/api/crypto/odp_crypto_test_inp.c @@ -1,4 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -174,7 +175,7 @@ static int alg_op(odp_packet_t pkt, odp_packet_data_range_t *cipher_range, odp_packet_data_range_t *auth_range, uint8_t *aad, - unsigned int plaintext_len) + unsigned int hash_result_offset) { int rc; odp_crypto_op_result_t result; @@ -198,7 +199,7 @@ static int alg_op(odp_packet_t pkt, op_params.aad_ptr = aad; - op_params.hash_result_offset = plaintext_len; + op_params.hash_result_offset = hash_result_offset; rc = odp_crypto_operation(&op_params, &posted, &result); if (rc < 0) { @@ -254,7 +255,7 @@ static int alg_packet_op(odp_packet_t pkt, odp_packet_data_range_t *cipher_range, odp_packet_data_range_t *auth_range, uint8_t *aad, - unsigned int plaintext_len) + unsigned int hash_result_offset) { int rc; odp_crypto_packet_result_t result; @@ -275,7 +276,7 @@ static int alg_packet_op(odp_packet_t pkt, op_params.aad_ptr = aad; - op_params.hash_result_offset = plaintext_len; + op_params.hash_result_offset = hash_result_offset; rc = odp_crypto_op(&pkt, &out_pkt, &op_params, 1); if (rc <= 0) { @@ -315,7 +316,7 @@ static int alg_packet_op_enq(odp_packet_t pkt, odp_packet_data_range_t *cipher_range, odp_packet_data_range_t *auth_range, uint8_t *aad, - unsigned int plaintext_len) + unsigned int hash_result_offset) { int rc; odp_event_t event; @@ -337,7 +338,7 @@ static int alg_packet_op_enq(odp_packet_t pkt, op_params.aad_ptr = aad; - op_params.hash_result_offset = plaintext_len; + op_params.hash_result_offset = hash_result_offset; rc = odp_crypto_op_enq(&pkt, &pkt, &op_params, 1); if (rc <= 0) { @@ -381,6 +382,69 @@ static int alg_packet_op_enq(odp_packet_t pkt, return 0; } +/* + * Try to adjust packet so that the first segment holds 'first_seg_len' bytes + * of packet data (+ tailroom if first_seg_len is longer than the packet). + * + * If 'first_seg_len' is zero, do not try to add segments but make headroom + * zero. + * + * Packet data bytes are not preserved. + */ +static void adjust_segments(odp_packet_t *pkt, uint32_t first_seg_len) +{ + uint32_t shift; + + shift = odp_packet_headroom(*pkt) + first_seg_len; + + if (odp_packet_extend_head(pkt, shift, NULL, NULL) < 0) { + CU_FAIL("odp_packet_extend_head() failed\n"); + return; + } + if (odp_packet_trunc_tail(pkt, shift, NULL, NULL) < 0) { + CU_FAIL("odp_packet_trunc_tail() failed\n"); + return; + } + /* + * ODP API does not seem to guarantee that we ever have a multi-segment + * packet at this point, but we can print a message about it. + */ + if (first_seg_len == 1 && + first_seg_len != odp_packet_seg_len(*pkt)) + printf("Could not create a segmented packet for testing.\n"); +} + +/* + * Generate or verify header and trailer bytes + */ +static void do_header_and_trailer(odp_packet_t pkt, + uint32_t header_len, uint32_t trailer_len, + odp_bool_t check) +{ + uint32_t trailer_offset = odp_packet_len(pkt) - trailer_len; + uint32_t max_len = header_len > trailer_len ? header_len : trailer_len; + uint8_t buffer[max_len]; + uint32_t n; + int rc; + + for (n = 0; n < max_len; n++) + buffer[n] = n; + + if (check) { + CU_ASSERT(!packet_cmp_mem_bytes(pkt, 0, + buffer, header_len)); + CU_ASSERT(!packet_cmp_mem_bytes(pkt, trailer_offset, + buffer, trailer_len)); + } else { + rc = odp_packet_copy_from_mem(pkt, 0, + header_len, buffer); + CU_ASSERT(rc == 0); + rc = odp_packet_copy_from_mem(pkt, trailer_offset, + trailer_len, buffer); + CU_ASSERT(rc == 0); + } +} + typedef enum crypto_test { NORMAL_TEST = 0, /**< Plain execution */ REPEAT_TEST, /**< Rerun without reinitializing the session */ @@ -388,6 +452,135 @@ typedef enum crypto_test { MAX_TEST, /**< Final mark */ } crypto_test; +typedef struct alg_test_param_t { + odp_crypto_session_t session; + odp_crypto_op_t op; + odp_auth_alg_t auth_alg; + crypto_test_reference_t *ref; + odp_bool_t override_iv; + odp_bool_t bit_mode; + odp_bool_t adjust_segmentation; + uint32_t first_seg_len; + uint32_t header_len; + uint32_t trailer_len; +} alg_test_param_t; + +static void alg_test_execute(const alg_test_param_t *param) +{ + int rc; + odp_bool_t ok = false; + int iteration; + uint32_t reflength; + odp_packet_data_range_t cipher_range; + odp_packet_data_range_t auth_range; + crypto_test_reference_t *ref = param->ref; + uint8_t *cipher_iv = param->override_iv ? ref->cipher_iv : NULL; + uint8_t *auth_iv = param->override_iv ? ref->auth_iv : NULL; + + cipher_range.offset = param->header_len; + cipher_range.length = ref->length; + auth_range.offset = param->header_len; + auth_range.length = ref->length; + + if (param->bit_mode) { + reflength = (ref->length + 7) / 8; + cipher_range.offset *= 8; + auth_range.offset *= 8; + } else { + reflength = ref->length; + } + + for (iteration = NORMAL_TEST; iteration < MAX_TEST; iteration++) { + odp_packet_t pkt; + uint32_t digest_offset = param->header_len + reflength; + + /* + * Test detection of wrong digest value in input packet + * only when decoding and using non-null auth algorithm. + */ + if (iteration == WRONG_DIGEST_TEST && + (param->auth_alg == ODP_AUTH_ALG_NULL || + param->op == ODP_CRYPTO_OP_ENCODE)) + continue; + + pkt = odp_packet_alloc(suite_context.pool, + param->header_len + reflength + + ref->digest_length + param->trailer_len); + CU_ASSERT(pkt != ODP_PACKET_INVALID); + if (pkt == ODP_PACKET_INVALID) + continue; + + if (param->adjust_segmentation) + adjust_segments(&pkt, param->first_seg_len); + + do_header_and_trailer(pkt, param->header_len, param->trailer_len, false); + + if (param->op == ODP_CRYPTO_OP_ENCODE) { + odp_packet_copy_from_mem(pkt, param->header_len, + reflength, ref->plaintext); + } else { + odp_packet_copy_from_mem(pkt, param->header_len, + reflength, ref->ciphertext); + odp_packet_copy_from_mem(pkt, digest_offset, + ref->digest_length, + ref->digest); + if (iteration == WRONG_DIGEST_TEST) { + uint8_t byte = ~ref->digest[0]; + + odp_packet_copy_from_mem(pkt, digest_offset, + 1, &byte); + } + } + + if (!suite_context.packet) + rc = alg_op(pkt, &ok, param->session, + cipher_iv, auth_iv, + &cipher_range, &auth_range, + ref->aad, digest_offset); + else if (ODP_CRYPTO_ASYNC == suite_context.op_mode) + rc = alg_packet_op_enq(pkt, &ok, param->session, + cipher_iv, auth_iv, + &cipher_range, &auth_range, + ref->aad, digest_offset); + else + rc = alg_packet_op(pkt, &ok, param->session, + cipher_iv, auth_iv, + &cipher_range, &auth_range, + ref->aad, digest_offset); + if (rc < 0) { + odp_packet_free(pkt); + break; + } + + if (iteration == WRONG_DIGEST_TEST) { + CU_ASSERT(!ok); + odp_packet_free(pkt); + continue; + } + + CU_ASSERT(ok); + + do_header_and_trailer(pkt, param->header_len, param->trailer_len, true); + + if (param->op == ODP_CRYPTO_OP_ENCODE) { + CU_ASSERT(!packet_cmp_mem(pkt, param->header_len, + ref->ciphertext, + ref->length, + param->bit_mode)); + CU_ASSERT(!packet_cmp_mem(pkt, digest_offset, + ref->digest, + ref->digest_length, + param->bit_mode)); + } else { + CU_ASSERT(!packet_cmp_mem(pkt, param->header_len, + ref->plaintext, + ref->length, + param->bit_mode)); + } + odp_packet_free(pkt); + } +} + /* Basic algorithm run function for async inplace mode. * Creates a session from input parameters and runs one operation * on input_vec. Checks the output of the crypto operation against @@ -404,15 +597,14 @@ static void alg_test(odp_crypto_op_t op, odp_bool_t ovr_iv, odp_bool_t bit_mode) { + unsigned int initial_num_failures = CU_get_number_of_failures(); odp_crypto_session_t session; int rc; - odp_crypto_ses_create_err_t status; - odp_bool_t ok = false; - int iteration; uint32_t reflength; + uint32_t seg_len; + uint32_t max_shift; + odp_crypto_ses_create_err_t status; odp_crypto_session_param_t ses_params; - odp_packet_data_range_t cipher_range; - odp_packet_data_range_t auth_range; odp_crypto_key_t cipher_key = { .data = ref->cipher_key, .length = ref->cipher_key_length @@ -429,6 +621,7 @@ static void alg_test(odp_crypto_op_t op, .data = ovr_iv ? NULL : ref->auth_iv, .length = ref->auth_iv_length }; + alg_test_param_t test_param; /* Create a crypto session */ odp_crypto_session_param_init(&ses_params); @@ -453,96 +646,47 @@ static void alg_test(odp_crypto_op_t op, CU_ASSERT(odp_crypto_session_to_u64(session) != odp_crypto_session_to_u64(ODP_CRYPTO_SESSION_INVALID)); - cipher_range.offset = 0; - cipher_range.length = ref->length; - auth_range.offset = 0; - auth_range.length = ref->length; + memset(&test_param, 0, sizeof(test_param)); + test_param.session = session; + test_param.op = op; + test_param.auth_alg = auth_alg; + test_param.ref = ref; + test_param.override_iv = ovr_iv; + test_param.bit_mode = bit_mode; + + alg_test_execute(&test_param); if (bit_mode) reflength = (ref->length + 7) / 8; else reflength = ref->length; - - /* Prepare input data */ - odp_packet_t pkt = odp_packet_alloc(suite_context.pool, - reflength + ref->digest_length); - CU_ASSERT(pkt != ODP_PACKET_INVALID); - if (pkt == ODP_PACKET_INVALID) - goto cleanup; - - for (iteration = NORMAL_TEST; iteration < MAX_TEST; iteration++) { - /* checking against wrong digest is meaningless for NULL digest - * or when generating digest */ - if (iteration == WRONG_DIGEST_TEST && - (auth_alg == ODP_AUTH_ALG_NULL || - op == ODP_CRYPTO_OP_ENCODE)) - continue; - - if (op == ODP_CRYPTO_OP_ENCODE) { - odp_packet_copy_from_mem(pkt, 0, reflength, - ref->plaintext); - } else { - odp_packet_copy_from_mem(pkt, 0, reflength, - ref->ciphertext); - odp_packet_copy_from_mem(pkt, reflength, - ref->digest_length, - ref->digest); - if (iteration == WRONG_DIGEST_TEST) { - uint8_t byte = ~ref->digest[0]; - - odp_packet_copy_from_mem(pkt, reflength, - 1, &byte); - } - } - - if (!suite_context.packet) - rc = alg_op(pkt, &ok, session, - ovr_iv ? ref->cipher_iv : NULL, - ovr_iv ? ref->auth_iv : NULL, - &cipher_range, &auth_range, - ref->aad, reflength); - else if (ODP_CRYPTO_ASYNC == suite_context.op_mode) - rc = alg_packet_op_enq(pkt, &ok, session, - ovr_iv ? ref->cipher_iv : NULL, - ovr_iv ? ref->auth_iv : NULL, - &cipher_range, &auth_range, - ref->aad, reflength); - else - rc = alg_packet_op(pkt, &ok, session, - ovr_iv ? ref->cipher_iv : NULL, - ovr_iv ? ref->auth_iv : NULL, - &cipher_range, &auth_range, - ref->aad, reflength); - if (rc < 0) + max_shift = reflength + ref->digest_length; + + /* + * Test with segmented packets with all possible segment boundaries + * within the packet data (including boundary after the packet data + * in the location where the digest will be written). + */ + for (seg_len = 0; seg_len <= max_shift; seg_len++) { + /* + * CUnit chokes on too many assertion failures, so bail + * out if this test has already failed. + */ + if (CU_get_number_of_failures() > initial_num_failures) break; - if (iteration == WRONG_DIGEST_TEST) { - CU_ASSERT(!ok); - continue; - } - - CU_ASSERT(ok); + test_param.adjust_segmentation = true; + test_param.first_seg_len = seg_len; + test_param.header_len = 0; + test_param.trailer_len = 0; + alg_test_execute(&test_param); - if (op == ODP_CRYPTO_OP_ENCODE) { - CU_ASSERT(!packet_cmp_mem(pkt, 0, - ref->ciphertext, - ref->length, - bit_mode)); - CU_ASSERT(!packet_cmp_mem(pkt, reflength, - ref->digest, - ref->digest_length, - bit_mode)); - } else { - CU_ASSERT(!packet_cmp_mem(pkt, 0, - ref->plaintext, - ref->length, - bit_mode)); - } + /* Test partial packet crypto with odd alignment. */ + test_param.header_len = 3; + test_param.trailer_len = 32; + alg_test_execute(&test_param); } - odp_packet_free(pkt); - -cleanup: rc = odp_crypto_session_destroy(session); CU_ASSERT(!rc); } @@ -1035,10 +1179,6 @@ static int check_alg_3des_cbc(void) return check_alg_support(ODP_CIPHER_ALG_3DES_CBC, ODP_AUTH_ALG_NULL); } -/* This test verifies the correctness of encode (plaintext -> ciphertext) - * operation for 3DES_CBC algorithm. IV for the operation is the session IV. - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer.*/ static void crypto_test_enc_alg_3des_cbc(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1050,9 +1190,6 @@ static void crypto_test_enc_alg_3des_cbc(void) false); } -/* This test verifies the correctness of encode (plaintext -> ciphertext) - * operation for 3DES_CBC algorithm. IV for the operation is the operation IV. - * */ static void crypto_test_enc_alg_3des_cbc_ovr_iv(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1064,11 +1201,6 @@ static void crypto_test_enc_alg_3des_cbc_ovr_iv(void) false); } -/* This test verifies the correctness of decode (ciphertext -> plaintext) - * operation for 3DES_CBC algorithm. IV for the operation is the session IV - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_dec_alg_3des_cbc(void) { check_alg(ODP_CRYPTO_OP_DECODE, @@ -1080,11 +1212,6 @@ static void crypto_test_dec_alg_3des_cbc(void) false); } -/* This test verifies the correctness of decode (ciphertext -> plaintext) - * operation for 3DES_CBC algorithm. IV for the operation is the session IV - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_dec_alg_3des_cbc_ovr_iv(void) { check_alg(ODP_CRYPTO_OP_DECODE, @@ -1101,8 +1228,6 @@ static int check_alg_3des_ecb(void) return check_alg_support(ODP_CIPHER_ALG_3DES_ECB, ODP_AUTH_ALG_NULL); } -/* This test verifies the correctness of encode (plaintext -> ciphertext) - * operation for 3DES_ECB algorithm. */ static void crypto_test_enc_alg_3des_ecb(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1114,11 +1239,6 @@ static void crypto_test_enc_alg_3des_ecb(void) false); } -/* This test verifies the correctness of decode (ciphertext -> plaintext) - * operation for 3DES_ECB algorithm. IV for the operation is the session IV - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_dec_alg_3des_ecb(void) { check_alg(ODP_CRYPTO_OP_DECODE, @@ -1185,10 +1305,6 @@ static int check_alg_aes_gcm(void) return check_alg_support(ODP_CIPHER_ALG_AES_GCM, ODP_AUTH_ALG_AES_GCM); } -/* This test verifies the correctness of encode (plaintext -> ciphertext) - * operation for AES128_GCM algorithm. IV for the operation is the session IV. - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer.*/ static void crypto_test_enc_alg_aes_gcm(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1200,10 +1316,6 @@ static void crypto_test_enc_alg_aes_gcm(void) false); } -/* This test verifies the correctness of encode (plaintext -> ciphertext) - * operation for AES128_GCM algorithm. IV for the operation is the session IV. - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer.*/ static void crypto_test_enc_alg_aes_gcm_ovr_iv(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1215,11 +1327,6 @@ static void crypto_test_enc_alg_aes_gcm_ovr_iv(void) false); } -/* This test verifies the correctness of decode (ciphertext -> plaintext) - * operation for 3DES_CBC algorithm. IV for the operation is the session IV - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_dec_alg_aes_gcm(void) { check_alg(ODP_CRYPTO_OP_DECODE, @@ -1231,11 +1338,6 @@ static void crypto_test_dec_alg_aes_gcm(void) false); } -/* This test verifies the correctness of decode (ciphertext -> plaintext) - * operation for 3DES_CBC algorithm. IV for the operation is the session IV - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_dec_alg_aes_gcm_ovr_iv(void) { check_alg(ODP_CRYPTO_OP_DECODE, @@ -1301,10 +1403,6 @@ static int check_alg_aes_cbc(void) return check_alg_support(ODP_CIPHER_ALG_AES_CBC, ODP_AUTH_ALG_NULL); } -/* This test verifies the correctness of encode (plaintext -> ciphertext) - * operation for AES128_CBC algorithm. IV for the operation is the session IV. - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer.*/ static void crypto_test_enc_alg_aes_cbc(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1316,9 +1414,6 @@ static void crypto_test_enc_alg_aes_cbc(void) false); } -/* This test verifies the correctness of encode (plaintext -> ciphertext) - * operation for AES128_CBC algorithm. IV for the operation is the operation IV. - * */ static void crypto_test_enc_alg_aes_cbc_ovr_iv(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1330,11 +1425,6 @@ static void crypto_test_enc_alg_aes_cbc_ovr_iv(void) false); } -/* This test verifies the correctness of decode (ciphertext -> plaintext) - * operation for AES128_CBC algorithm. IV for the operation is the session IV - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_dec_alg_aes_cbc(void) { check_alg(ODP_CRYPTO_OP_DECODE, @@ -1346,11 +1436,6 @@ static void crypto_test_dec_alg_aes_cbc(void) false); } -/* This test verifies the correctness of decode (ciphertext -> plaintext) - * operation for AES128_CBC algorithm. IV for the operation is the session IV - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_dec_alg_aes_cbc_ovr_iv(void) { check_alg(ODP_CRYPTO_OP_DECODE, @@ -1367,10 +1452,6 @@ static int check_alg_aes_ctr(void) return check_alg_support(ODP_CIPHER_ALG_AES_CTR, ODP_AUTH_ALG_NULL); } -/* This test verifies the correctness of encode (plaintext -> ciphertext) - * operation for AES128_CTR algorithm. IV for the operation is the session IV. - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer.*/ static void crypto_test_enc_alg_aes_ctr(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1382,9 +1463,6 @@ static void crypto_test_enc_alg_aes_ctr(void) false); } -/* This test verifies the correctness of encode (plaintext -> ciphertext) - * operation for AES128_CTR algorithm. IV for the operation is the operation IV. - * */ static void crypto_test_enc_alg_aes_ctr_ovr_iv(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1396,11 +1474,6 @@ static void crypto_test_enc_alg_aes_ctr_ovr_iv(void) false); } -/* This test verifies the correctness of decode (ciphertext -> plaintext) - * operation for AES128_CTR algorithm. IV for the operation is the session IV - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_dec_alg_aes_ctr(void) { check_alg(ODP_CRYPTO_OP_DECODE, @@ -1412,11 +1485,6 @@ static void crypto_test_dec_alg_aes_ctr(void) false); } -/* This test verifies the correctness of decode (ciphertext -> plaintext) - * operation for AES128_CTR algorithm. IV for the operation is the session IV - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_dec_alg_aes_ctr_ovr_iv(void) { check_alg(ODP_CRYPTO_OP_DECODE, @@ -1433,9 +1501,6 @@ static int check_alg_aes_ecb(void) return check_alg_support(ODP_CIPHER_ALG_AES_ECB, ODP_AUTH_ALG_NULL); } -/* This test verifies the correctness of encode (plaintext -> ciphertext) - * operation for AES128_ECB algorithm. - */ static void crypto_test_enc_alg_aes_ecb(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1447,9 +1512,6 @@ static void crypto_test_enc_alg_aes_ecb(void) false); } -/* This test verifies the correctness of decode (ciphertext -> plaintext) - * operation for AES128_ECB algorithm. - * */ static void crypto_test_dec_alg_aes_ecb(void) { check_alg(ODP_CRYPTO_OP_DECODE, @@ -1466,11 +1528,6 @@ static int check_alg_aes_cfb128(void) return check_alg_support(ODP_CIPHER_ALG_AES_CFB128, ODP_AUTH_ALG_NULL); } -/* This test verifies the correctness of encode (plaintext -> ciphertext) - * operation for AES128_CFB128 algorithm. IV for the operation is the session - * IV. - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer.*/ static void crypto_test_enc_alg_aes_cfb128(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1482,10 +1539,6 @@ static void crypto_test_enc_alg_aes_cfb128(void) false); } -/* This test verifies the correctness of encode (plaintext -> ciphertext) - * operation for AES128_CFB128 algorithm. IV for the operation is the operation - * IV. - * */ static void crypto_test_enc_alg_aes_cfb128_ovr_iv(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1497,11 +1550,6 @@ static void crypto_test_enc_alg_aes_cfb128_ovr_iv(void) false); } -/* This test verifies the correctness of decode (ciphertext -> plaintext) - * operation for AES128_CFB128 algorithm. IV for the operation is the session IV - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_dec_alg_aes_cfb128(void) { check_alg(ODP_CRYPTO_OP_DECODE, @@ -1513,11 +1561,6 @@ static void crypto_test_dec_alg_aes_cfb128(void) false); } -/* This test verifies the correctness of decode (ciphertext -> plaintext) - * operation for AES128_CFB128 algorithm. IV for the operation is the session IV - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_dec_alg_aes_cfb128_ovr_iv(void) { check_alg(ODP_CRYPTO_OP_DECODE, @@ -1534,10 +1577,6 @@ static int check_alg_aes_xts(void) return check_alg_support(ODP_CIPHER_ALG_AES_XTS, ODP_AUTH_ALG_NULL); } -/* This test verifies the correctness of encode (plaintext -> ciphertext) - * operation for AES128_XTS algorithm. IV for the operation is the session IV. - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer.*/ static void crypto_test_enc_alg_aes_xts(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1549,9 +1588,6 @@ static void crypto_test_enc_alg_aes_xts(void) false); } -/* This test verifies the correctness of encode (plaintext -> ciphertext) - * operation for AES128_XTS algorithm. IV for the operation is the operation IV. - * */ static void crypto_test_enc_alg_aes_xts_ovr_iv(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1563,11 +1599,6 @@ static void crypto_test_enc_alg_aes_xts_ovr_iv(void) false); } -/* This test verifies the correctness of decode (ciphertext -> plaintext) - * operation for AES128_XTS algorithm. IV for the operation is the session IV - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_dec_alg_aes_xts(void) { check_alg(ODP_CRYPTO_OP_DECODE, @@ -1579,11 +1610,6 @@ static void crypto_test_dec_alg_aes_xts(void) false); } -/* This test verifies the correctness of decode (ciphertext -> plaintext) - * operation for AES128_XTS algorithm. IV for the operation is the session IV - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_dec_alg_aes_xts_ovr_iv(void) { check_alg(ODP_CRYPTO_OP_DECODE, @@ -1797,13 +1823,6 @@ static int check_alg_hmac_md5(void) return check_alg_support(ODP_CIPHER_ALG_NULL, ODP_AUTH_ALG_MD5_HMAC); } -/* This test verifies the correctness of HMAC_MD5 digest operation. - * The output check length is truncated to 12 bytes (96 bits) as - * returned by the crypto operation API call. - * Note that hash digest is a one-way operation. - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_gen_alg_hmac_md5(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1831,13 +1850,6 @@ static int check_alg_hmac_sha1(void) return check_alg_support(ODP_CIPHER_ALG_NULL, ODP_AUTH_ALG_SHA1_HMAC); } -/* This test verifies the correctness of HMAC_SHA1 digest operation. - * The output check length is truncated to 12 bytes (96 bits) as - * returned by the crypto operation API call. - * Note that hash digest is a one-way operation. - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_gen_alg_hmac_sha1(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1892,13 +1904,6 @@ static int check_alg_hmac_sha256(void) return check_alg_support(ODP_CIPHER_ALG_NULL, ODP_AUTH_ALG_SHA256_HMAC); } -/* This test verifies the correctness of HMAC_SHA256 digest operation. - * The output check length is truncated to 16 bytes (128 bits) as - * returned by the crypto operation API call. - * Note that hash digest is a one-way operation. - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_gen_alg_hmac_sha256(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1926,13 +1931,6 @@ static int check_alg_hmac_sha384(void) return check_alg_support(ODP_CIPHER_ALG_NULL, ODP_AUTH_ALG_SHA384_HMAC); } -/* This test verifies the correctness of HMAC_SHA384 digest operation. - * The output check length is truncated to 24 bytes (192 bits) as - * returned by the crypto operation API call. - * Note that hash digest is a one-way operation. - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_gen_alg_hmac_sha384(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1960,13 +1958,6 @@ static int check_alg_hmac_sha512(void) return check_alg_support(ODP_CIPHER_ALG_NULL, ODP_AUTH_ALG_SHA512_HMAC); } -/* This test verifies the correctness of HMAC_SHA512 digest operation. - * The output check length is truncated to 32 bytes (256 bits) as - * returned by the crypto operation API call. - * Note that hash digest is a one-way operation. - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_gen_alg_hmac_sha512(void) { check_alg(ODP_CRYPTO_OP_ENCODE, @@ -1995,13 +1986,6 @@ static int check_alg_aes_xcbc(void) ODP_AUTH_ALG_AES_XCBC_MAC); } -/* This test verifies the correctness of AES_XCBC_MAC digest operation. - * The output check length is truncated to 16 bytes (128 bits) as - * returned by the crypto operation API call. - * Note that hash digest is a one-way operation. - * In addition the test verifies if the implementation can use the - * packet buffer as completion event buffer. - * */ static void crypto_test_gen_alg_aes_xcbc(void) { check_alg(ODP_CRYPTO_OP_ENCODE, diff --git a/test/validation/api/ipsec/Makefile.am b/test/validation/api/ipsec/Makefile.am index 8417a5776..1a29d687d 100644 --- a/test/validation/api/ipsec/Makefile.am +++ b/test/validation/api/ipsec/Makefile.am @@ -7,7 +7,8 @@ libtestipsec_la_SOURCES = \ ipsec_test_in.c \ ipsec_test_out.c \ ipsec.h \ - ipsec.c + ipsec.c \ + reass_test_vectors.c test_PROGRAMS = \ ipsec_sync \ diff --git a/test/validation/api/ipsec/ipsec.c b/test/validation/api/ipsec/ipsec.c index 5bbc9c025..c6fbe6f91 100644 --- a/test/validation/api/ipsec/ipsec.c +++ b/test/validation/api/ipsec/ipsec.c @@ -20,6 +20,7 @@ struct suite_context_s suite_context; static odp_ipsec_capability_t capa; #define PKT_POOL_NUM 64 +#define EVENT_WAIT_TIME ODP_TIME_SEC_IN_NS #define PACKET_USER_PTR ((void *)0x1212fefe) #define IPSEC_SA_CTX ((void *)0xfefefafa) @@ -100,6 +101,41 @@ static int pktio_start(odp_pktio_t pktio, odp_bool_t in, odp_bool_t out) return 1; } +static odp_event_t sched_queue_deq(uint64_t wait_ns) +{ + uint64_t wait = odp_schedule_wait_time(wait_ns); + + return odp_schedule(NULL, wait); +} + +static odp_event_t plain_queue_deq(odp_queue_t queue, uint64_t wait_ns) +{ + odp_time_t cur, wait, next; + odp_event_t event; + + wait = odp_time_local_from_ns(wait_ns); + next = odp_time_sum(odp_time_local(), wait); + + do { + event = odp_queue_deq(queue); + cur = odp_time_local(); + } while (event == ODP_EVENT_INVALID && odp_time_cmp(next, cur) >= 0); + + return event; +} + +static odp_event_t recv_event(odp_queue_t queue, uint64_t wait_ns) +{ + odp_event_t event; + + if (odp_queue_type(queue) == ODP_QUEUE_TYPE_PLAIN) + event = plain_queue_deq(queue, wait_ns); + else + event = sched_queue_deq(wait_ns); + + return event; +} + static void pktio_stop(odp_pktio_t pktio) { odp_queue_t queue = ODP_QUEUE_INVALID; @@ -110,7 +146,7 @@ static void pktio_stop(odp_pktio_t pktio) fprintf(stderr, "IPsec pktio stop failed.\n"); while (1) { - odp_event_t ev = odp_queue_deq(queue); + odp_event_t ev = recv_event(queue, 0); if (ev != ODP_EVENT_INVALID) odp_event_free(ev); @@ -139,6 +175,17 @@ int ipsec_check(odp_bool_t ah, ODP_SUPPORT_NO == capa.op_mode_inline_out)) return ODP_TEST_INACTIVE; + if (!(ODP_IPSEC_OP_MODE_SYNC == suite_context.inbound_op_mode && + ODP_IPSEC_OP_MODE_SYNC == suite_context.outbound_op_mode) && + ODP_QUEUE_INVALID != suite_context.queue) { + if (suite_context.q_type == ODP_QUEUE_TYPE_PLAIN && + !capa.queue_type_plain) + return ODP_TEST_INACTIVE; + if (suite_context.q_type == ODP_QUEUE_TYPE_SCHED && + !capa.queue_type_sched) + return ODP_TEST_INACTIVE; + } + /* suite_context.pktio is set to ODP_PKTIO_INVALID in ipsec_suite_init() * if the pktio device doesn't support inline IPsec processing. */ if (suite_context.pktio == ODP_PKTIO_INVALID && @@ -356,9 +403,7 @@ void ipsec_sa_destroy(odp_ipsec_sa_t sa) CU_ASSERT_EQUAL(ODP_IPSEC_OK, odp_ipsec_sa_disable(sa)); if (ODP_QUEUE_INVALID != suite_context.queue) { - do { - event = odp_queue_deq(suite_context.queue); - } while (event == ODP_EVENT_INVALID); + event = recv_event(suite_context.queue, EVENT_WAIT_TIME); CU_ASSERT(odp_event_is_valid(event) == 1); CU_ASSERT_EQUAL(ODP_EVENT_IPSEC_STATUS, odp_event_type(event)); @@ -516,12 +561,27 @@ static int send_pkts(const ipsec_test_part part[], int num_part) return num_part; } +/* Receive async inbound packet */ +static odp_event_t recv_pkt_async_inbound(odp_ipsec_op_status_t status) +{ + odp_queue_t queue; + + /* + * In case of SA lookup failure, the event is enqueued to the default + * queue specified during odp_ipsec_config() + */ + if (status.error.sa_lookup == 0) + queue = suite_context.queue; + else + queue = suite_context.default_queue; + + return recv_event(queue, EVENT_WAIT_TIME); +} + /* Receive inline processed packets */ static int recv_pkts_inline(const ipsec_test_part *part, odp_packet_t *pkto) { - odp_packet_reass_partial_state_t reass_state; - odp_packet_reass_status_t reass_status; odp_queue_t queue = ODP_QUEUE_INVALID; int i; @@ -532,7 +592,7 @@ static int recv_pkts_inline(const ipsec_test_part *part, odp_event_t ev; odp_event_subtype_t subtype; - ev = odp_queue_deq(queue); + ev = recv_event(queue, 0); if (ODP_EVENT_INVALID != ev) { CU_ASSERT(odp_event_is_valid(ev) == 1); CU_ASSERT_EQUAL(ODP_EVENT_PACKET, @@ -547,10 +607,15 @@ static int recv_pkts_inline(const ipsec_test_part *part, continue; } - ev = odp_queue_deq(suite_context.queue); + ev = recv_event(suite_context.queue, 0); if (ODP_EVENT_INVALID != ev) { odp_packet_t pkt; int num_pkts = 0; + odp_packet_reass_status_t reass_status; + odp_packet_reass_info_t reass = {0}; + odp_packet_reass_partial_state_t reass_state; + odp_packet_t frags[MAX_FRAGS]; + int j; CU_ASSERT(odp_event_is_valid(ev) == 1); CU_ASSERT_EQUAL(ODP_EVENT_PACKET, odp_event_type(ev)); @@ -559,13 +624,18 @@ static int recv_pkts_inline(const ipsec_test_part *part, CU_ASSERT(!part->out[i].status.error.sa_lookup); reass_status = odp_packet_reass_status(pkt); - if (ODP_PACKET_REASS_INCOMPLETE != reass_status) { + CU_ASSERT(reass_status == part->out[i].reass_status); + + switch (reass_status) { + case ODP_PACKET_REASS_COMPLETE: + CU_ASSERT(odp_packet_reass_info(pkt, &reass) == 0); + CU_ASSERT(part->out[i].num_frags == reass.num_frags); + /* FALLTHROUGH */ + case ODP_PACKET_REASS_NONE: pkto[i] = pkt; num_pkts = 1; - } else { - odp_packet_t frags[MAX_FRAGS]; - int j; - + break; + case ODP_PACKET_REASS_INCOMPLETE: CU_ASSERT(0 == odp_packet_reass_partial_state(pkt, frags, &reass_state)); num_pkts = reass_state.num_frags; @@ -573,6 +643,10 @@ static int recv_pkts_inline(const ipsec_test_part *part, CU_ASSERT_FATAL(i + num_pkts <= part->num_pkt); for (j = 0; j < num_pkts; j++) pkto[i + j] = frags[j]; + break; + default: + CU_FAIL("Unknown reassembly status"); + break; } for (; num_pkts > 0; num_pkts--) @@ -625,9 +699,7 @@ static int ipsec_process_in(const ipsec_test_part *part, odp_event_t event; odp_event_subtype_t subtype; - do { - event = odp_queue_deq(suite_context.queue); - } while (event == ODP_EVENT_INVALID); + event = recv_pkt_async_inbound(part->out[i].status); CU_ASSERT(odp_event_is_valid(event) == 1); CU_ASSERT_EQUAL(ODP_EVENT_PACKET, @@ -680,9 +752,7 @@ static int ipsec_send_out_one(const ipsec_test_part *part, odp_event_t event; odp_event_subtype_t subtype; - do { - event = odp_queue_deq(suite_context.queue); - } while (event == ODP_EVENT_INVALID); + event = recv_event(suite_context.queue, EVENT_WAIT_TIME); CU_ASSERT(odp_event_is_valid(event) == 1); CU_ASSERT_EQUAL(ODP_EVENT_PACKET, @@ -764,7 +834,7 @@ static int ipsec_send_out_one(const ipsec_test_part *part, odp_event_t ev; odp_event_subtype_t subtype; - ev = odp_queue_deq(queue); + ev = recv_event(queue, 0); if (ODP_EVENT_INVALID != ev) { CU_ASSERT(odp_event_is_valid(ev) == 1); CU_ASSERT_EQUAL(ODP_EVENT_PACKET, @@ -779,7 +849,7 @@ static int ipsec_send_out_one(const ipsec_test_part *part, continue; } - ev = odp_queue_deq(suite_context.queue); + ev = recv_event(suite_context.queue, 0); if (ODP_EVENT_INVALID != ev) { CU_ASSERT(odp_event_is_valid(ev) == 1); CU_ASSERT_EQUAL(ODP_EVENT_PACKET, @@ -878,6 +948,11 @@ static void verify_in(const ipsec_test_part *part, if (ODP_IPSEC_SA_INVALID != sa) CU_ASSERT_EQUAL(IPSEC_SA_CTX, odp_ipsec_sa_context(sa)); + if (suite_context.inbound_op_mode != ODP_IPSEC_OP_MODE_SYNC) { + uint32_t len = part->pkt_in->len - part->pkt_in->l3_offset; + + CU_ASSERT(result.orig_ip_len == len); + } } ipsec_check_packet(part->out[i].pkt_res, pkto[i], @@ -1005,7 +1080,7 @@ void ipsec_check_out_one(const ipsec_test_part *part, odp_ipsec_sa_t sa) odp_packet_free(pkto[i]); } -int ipsec_suite_init(void) +static int ipsec_suite_init(void) { int rc = 0; @@ -1033,11 +1108,16 @@ void ipsec_test_packet_from_pkt(ipsec_test_packet *test_pkt, odp_packet_t *pkt) odp_packet_free(*pkt); } -static int ipsec_suite_term(void) +int ipsec_suite_term(void) { if (suite_context.pktio != ODP_PKTIO_INVALID) pktio_stop(suite_context.pktio); + if (ODP_QUEUE_INVALID != suite_context.queue) { + if (odp_queue_destroy(suite_context.queue)) + fprintf(stderr, "IPsec destq destroy failed.\n"); + } + if (odp_cunit_print_inactive()) return -1; @@ -1054,19 +1134,80 @@ int ipsec_out_term(void) return ipsec_suite_term(); } +static odp_queue_t sched_queue_create(const char *name) +{ + odp_queue_param_t qparam; + + odp_queue_param_init(&qparam); + qparam.type = ODP_QUEUE_TYPE_SCHED; + qparam.sched.prio = ODP_SCHED_PRIO_DEFAULT; + qparam.sched.sync = ODP_SCHED_SYNC_PARALLEL; + qparam.sched.group = ODP_SCHED_GROUP_ALL; + + return odp_queue_create(name, &qparam); +} + +static odp_queue_t plain_queue_create(const char *name) +{ + return odp_queue_create(name, NULL); +} + +int ipsec_suite_sync_init(void) +{ + suite_context.queue = ODP_QUEUE_INVALID; + + /* q_type doesn't matter when queue handle is invalid. */ + suite_context.q_type = ODP_QUEUE_TYPE_PLAIN; + + return ipsec_suite_init(); +} + +int ipsec_suite_plain_init(void) +{ + odp_queue_t dest_queue; + + dest_queue = plain_queue_create("ipsec-out"); + if (ODP_QUEUE_INVALID == dest_queue) { + fprintf(stderr, "IPsec destq creation failed.\n"); + return -1; + } + + suite_context.queue = dest_queue; + suite_context.q_type = ODP_QUEUE_TYPE_PLAIN; + + return ipsec_suite_init(); +} + +int ipsec_suite_sched_init(void) +{ + odp_queue_t dest_queue; + + dest_queue = sched_queue_create("ipsec-out"); + if (ODP_QUEUE_INVALID == dest_queue) { + fprintf(stderr, "IPsec destq creation failed.\n"); + return -1; + } + + suite_context.queue = dest_queue; + suite_context.q_type = ODP_QUEUE_TYPE_SCHED; + + return ipsec_suite_init(); +} + int ipsec_init(odp_instance_t *inst, odp_ipsec_op_mode_t mode) { odp_pool_param_t params; odp_pool_t pool; - odp_queue_t out_queue; odp_pool_capability_t pool_capa; odp_pktio_t pktio; odp_init_t init_param; odph_helper_options_t helper_options; + suite_context.reass_ipv4 = false; + suite_context.reass_ipv6 = false; suite_context.pool = ODP_POOL_INVALID; - suite_context.queue = ODP_QUEUE_INVALID; suite_context.pktio = ODP_PKTIO_INVALID; + suite_context.default_queue = ODP_QUEUE_INVALID; if (odph_options(&helper_options)) { fprintf(stderr, "error: odph_options() failed.\n"); @@ -1086,6 +1227,11 @@ int ipsec_init(odp_instance_t *inst, odp_ipsec_op_mode_t mode) return -1; } + if (odp_schedule_config(NULL)) { + fprintf(stderr, "odp_schedule_config() failed.\n"); + return -1; + } + if (odp_pool_capability(&pool_capa) < 0) { fprintf(stderr, "error: odp_pool_capability() failed.\n"); return -1; @@ -1115,14 +1261,6 @@ int ipsec_init(odp_instance_t *inst, odp_ipsec_op_mode_t mode) fprintf(stderr, "Packet pool creation failed.\n"); return -1; } - if (mode == ODP_IPSEC_OP_MODE_ASYNC || - mode == ODP_IPSEC_OP_MODE_INLINE) { - out_queue = odp_queue_create("ipsec-out", NULL); - if (ODP_QUEUE_INVALID == out_queue) { - fprintf(stderr, "IPsec outq creation failed.\n"); - return -1; - } - } if (mode == ODP_IPSEC_OP_MODE_INLINE) { pktio = pktio_create(pool); @@ -1159,6 +1297,19 @@ int ipsec_config(odp_instance_t ODP_UNUSED inst) ODP_SUPPORT_NO == capa.op_mode_inline_out)) return 0; + if (suite_context.inbound_op_mode == ODP_IPSEC_OP_MODE_ASYNC || + suite_context.inbound_op_mode == ODP_IPSEC_OP_MODE_INLINE) { + if (capa.queue_type_plain) + suite_context.default_queue = plain_queue_create("ipsec-default"); + else if (capa.queue_type_sched) + suite_context.default_queue = sched_queue_create("ipsec-default"); + + if (ODP_QUEUE_INVALID == suite_context.default_queue) { + fprintf(stderr, "IPsec defaultq creation failed.\n"); + return -1; + } + } + reass_test_vectors_init(); odp_ipsec_config_init(&ipsec_config); @@ -1166,7 +1317,7 @@ int ipsec_config(odp_instance_t ODP_UNUSED inst) ipsec_config.inbound_mode = suite_context.inbound_op_mode; ipsec_config.outbound_mode = suite_context.outbound_op_mode; ipsec_config.outbound.all_chksum = ~0; - ipsec_config.inbound.default_queue = suite_context.queue; + ipsec_config.inbound.default_queue = suite_context.default_queue; ipsec_config.inbound.parse_level = ODP_PROTO_LAYER_ALL; ipsec_config.inbound.chksums.all_chksum = ~0; ipsec_config.stats_en = true; @@ -1242,7 +1393,7 @@ int ipsec_config(odp_instance_t ODP_UNUSED inst) int ipsec_term(odp_instance_t inst) { odp_pool_t pool = suite_context.pool; - odp_queue_t out_queue = suite_context.queue; + odp_queue_t default_queue = suite_context.default_queue; /* suite_context.pktio is set to ODP_PKTIO_INVALID by ipsec_suite_init() if inline processing is not supported. */ odp_pktio_t pktio = odp_pktio_lookup("loop"); @@ -1252,9 +1403,9 @@ int ipsec_term(odp_instance_t inst) fprintf(stderr, "IPsec pktio close failed.\n"); } - if (ODP_QUEUE_INVALID != out_queue) { - if (odp_queue_destroy(out_queue)) - fprintf(stderr, "IPsec outq destroy failed.\n"); + if (ODP_QUEUE_INVALID != default_queue) { + if (odp_queue_destroy(default_queue)) + fprintf(stderr, "IPsec defaultq destroy failed.\n"); } if (ODP_POOL_INVALID != pool) { diff --git a/test/validation/api/ipsec/ipsec.h b/test/validation/api/ipsec/ipsec.h index b899fce48..6edce03d0 100644 --- a/test/validation/api/ipsec/ipsec.h +++ b/test/validation/api/ipsec/ipsec.h @@ -29,7 +29,10 @@ int ipsec_config(odp_instance_t inst); int ipsec_in_inline_init(void); int ipsec_out_inline_init(void); -int ipsec_suite_init(void); +int ipsec_suite_sync_init(void); +int ipsec_suite_plain_init(void); +int ipsec_suite_sched_init(void); +int ipsec_suite_term(void); int ipsec_in_term(void); int ipsec_out_term(void); @@ -39,8 +42,10 @@ struct suite_context_s { odp_ipsec_op_mode_t inbound_op_mode; odp_ipsec_op_mode_t outbound_op_mode; odp_pool_t pool; + odp_queue_t default_queue; odp_queue_t queue; odp_pktio_t pktio; + odp_queue_type_t q_type; }; extern struct suite_context_s suite_context; @@ -85,6 +90,8 @@ typedef struct { int num_pkt; struct { odp_ipsec_op_status_t status; + odp_packet_reass_status_t reass_status; + uint16_t num_frags; const ipsec_test_packet *pkt_res; odp_proto_l3_type_t l3_type; odp_proto_l4_type_t l4_type; diff --git a/test/validation/api/ipsec/ipsec_async.c b/test/validation/api/ipsec/ipsec_async.c index 8f818a94e..44c968a4d 100644 --- a/test/validation/api/ipsec/ipsec_async.c +++ b/test/validation/api/ipsec/ipsec_async.c @@ -10,6 +10,9 @@ static int ipsec_async_init(odp_instance_t *inst) { int rc; + suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; + suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; + rc = ipsec_init(inst, ODP_IPSEC_OP_MODE_ASYNC); if (rc != 0) return rc; @@ -17,19 +20,19 @@ static int ipsec_async_init(odp_instance_t *inst) suite_context.pool = odp_pool_lookup("packet_pool"); if (suite_context.pool == ODP_POOL_INVALID) return -1; - suite_context.queue = odp_queue_lookup("ipsec-out"); - if (suite_context.queue == ODP_QUEUE_INVALID) - return -1; - - suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; - suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; return ipsec_config(*inst); } odp_suiteinfo_t ipsec_suites[] = { - {"IPsec-in", ipsec_suite_init, ipsec_in_term, ipsec_in_suite}, - {"IPsec-out", ipsec_suite_init, ipsec_out_term, ipsec_out_suite}, + {"IPsec-plain-in", ipsec_suite_plain_init, ipsec_suite_term, + ipsec_in_suite}, + {"IPsec-sched-in", ipsec_suite_sched_init, ipsec_suite_term, + ipsec_in_suite}, + {"IPsec-plain-out", ipsec_suite_plain_init, ipsec_suite_term, + ipsec_out_suite}, + {"IPsec-sched-out", ipsec_suite_sched_init, ipsec_suite_term, + ipsec_out_suite}, ODP_SUITE_INFO_NULL, }; diff --git a/test/validation/api/ipsec/ipsec_inline_in.c b/test/validation/api/ipsec/ipsec_inline_in.c index bd34f6cc4..cbb3a178a 100644 --- a/test/validation/api/ipsec/ipsec_inline_in.c +++ b/test/validation/api/ipsec/ipsec_inline_in.c @@ -10,6 +10,9 @@ static int ipsec_sync_init(odp_instance_t *inst) { int rc; + suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_INLINE; + suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; + rc = ipsec_init(inst, ODP_IPSEC_OP_MODE_INLINE); if (rc != 0) return rc; @@ -17,21 +20,18 @@ static int ipsec_sync_init(odp_instance_t *inst) suite_context.pool = odp_pool_lookup("packet_pool"); if (suite_context.pool == ODP_POOL_INVALID) return -1; - suite_context.queue = odp_queue_lookup("ipsec-out"); - if (suite_context.queue == ODP_QUEUE_INVALID) - return -1; suite_context.pktio = odp_pktio_lookup("loop"); if (suite_context.pktio == ODP_PKTIO_INVALID) return -1; - suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_INLINE; - suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; - return ipsec_config(*inst); } odp_suiteinfo_t ipsec_suites[] = { - {"IPsec-in", ipsec_suite_init, ipsec_in_term, ipsec_in_suite}, + {"IPsec-plain-in", ipsec_suite_plain_init, ipsec_suite_term, + ipsec_in_suite}, + {"IPsec-sched-in", ipsec_suite_sched_init, ipsec_suite_term, + ipsec_in_suite}, ODP_SUITE_INFO_NULL, }; diff --git a/test/validation/api/ipsec/ipsec_inline_out.c b/test/validation/api/ipsec/ipsec_inline_out.c index 5cda43b74..3da19892b 100644 --- a/test/validation/api/ipsec/ipsec_inline_out.c +++ b/test/validation/api/ipsec/ipsec_inline_out.c @@ -10,6 +10,9 @@ static int ipsec_sync_init(odp_instance_t *inst) { int rc; + suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; + suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_INLINE; + rc = ipsec_init(inst, ODP_IPSEC_OP_MODE_INLINE); if (rc != 0) return rc; @@ -17,21 +20,18 @@ static int ipsec_sync_init(odp_instance_t *inst) suite_context.pool = odp_pool_lookup("packet_pool"); if (suite_context.pool == ODP_POOL_INVALID) return -1; - suite_context.queue = odp_queue_lookup("ipsec-out"); - if (suite_context.queue == ODP_QUEUE_INVALID) - return -1; suite_context.pktio = odp_pktio_lookup("loop"); if (suite_context.pktio == ODP_PKTIO_INVALID) return -1; - suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; - suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_INLINE; - return ipsec_config(*inst); } odp_suiteinfo_t ipsec_suites[] = { - {"IPsec-out", ipsec_suite_init, ipsec_out_term, ipsec_out_suite}, + {"IPsec-plain-out", ipsec_suite_plain_init, ipsec_suite_term, + ipsec_out_suite}, + {"IPsec-sched-out", ipsec_suite_sched_init, ipsec_suite_term, + ipsec_out_suite}, ODP_SUITE_INFO_NULL, }; diff --git a/test/validation/api/ipsec/ipsec_sync.c b/test/validation/api/ipsec/ipsec_sync.c index f77bb1c51..74ae1fe19 100644 --- a/test/validation/api/ipsec/ipsec_sync.c +++ b/test/validation/api/ipsec/ipsec_sync.c @@ -10,6 +10,9 @@ static int ipsec_sync_init(odp_instance_t *inst) { int rc; + suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_SYNC; + suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_SYNC; + rc = ipsec_init(inst, ODP_IPSEC_OP_MODE_SYNC); if (rc != 0) return rc; @@ -18,15 +21,12 @@ static int ipsec_sync_init(odp_instance_t *inst) if (suite_context.pool == ODP_POOL_INVALID) return -1; - suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_SYNC; - suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_SYNC; - return ipsec_config(*inst); } odp_suiteinfo_t ipsec_suites[] = { - {"IPsec-in", ipsec_suite_init, ipsec_in_term, ipsec_in_suite}, - {"IPsec-out", ipsec_suite_init, ipsec_out_term, ipsec_out_suite}, + {"IPsec-in", ipsec_suite_sync_init, ipsec_in_term, ipsec_in_suite}, + {"IPsec-out", ipsec_suite_sync_init, ipsec_out_term, ipsec_out_suite}, ODP_SUITE_INFO_NULL, }; diff --git a/test/validation/api/ipsec/ipsec_test_in.c b/test/validation/api/ipsec/ipsec_test_in.c index ef6996f1d..bcdcd4e83 100644 --- a/test/validation/api/ipsec/ipsec_test_in.c +++ b/test/validation/api/ipsec/ipsec_test_in.c @@ -1772,7 +1772,8 @@ static void test_multi_out_in(odp_ipsec_sa_t out_sa, uint8_t tunnel_ip_ver, int num_input_packets, ipsec_test_packet *input_packets[], - ipsec_test_packet *result_packet) + ipsec_test_packet *result_packet, + odp_packet_reass_status_t reass_status) { uint8_t ver_ihl = result_packet->data[result_packet->l3_offset]; odp_bool_t is_result_ipv6 = (ODPH_IPV4HDR_VER(ver_ihl) == ODPH_IPV6); @@ -1800,6 +1801,8 @@ static void test_multi_out_in(odp_ipsec_sa_t out_sa, if (i == num_input_packets - 1) { part_prep_plain(&test_in, 1, is_result_ipv6, true); test_in.out[0].pkt_res = result_packet; + test_in.out[0].reass_status = reass_status; + test_in.out[0].num_frags = num_input_packets; } ipsec_test_packet_from_pkt(&test_pkt, &pkt); test_in.pkt_in = &test_pkt; @@ -1820,7 +1823,8 @@ static void test_in_ipv4_esp_reass_success_two_frags(odp_ipsec_sa_t out_sa, test_multi_out_in(out_sa, in_sa, ODPH_IPV4, ARRAY_SIZE(input_packets), input_packets, - result_packet); + result_packet, + ODP_PACKET_REASS_COMPLETE); } static void test_in_ipv4_esp_reass_success_four_frags(odp_ipsec_sa_t out_sa, @@ -1837,7 +1841,8 @@ static void test_in_ipv4_esp_reass_success_four_frags(odp_ipsec_sa_t out_sa, test_multi_out_in(out_sa, in_sa, ODPH_IPV4, ARRAY_SIZE(input_packets), input_packets, - result_packet); + result_packet, + ODP_PACKET_REASS_COMPLETE); } static void test_in_ipv4_esp_reass_success_two_frags_ooo(odp_ipsec_sa_t out_sa, @@ -1852,7 +1857,8 @@ static void test_in_ipv4_esp_reass_success_two_frags_ooo(odp_ipsec_sa_t out_sa, test_multi_out_in(out_sa, in_sa, ODPH_IPV4, ARRAY_SIZE(input_packets), input_packets, - result_packet); + result_packet, + ODP_PACKET_REASS_COMPLETE); } static void test_in_ipv4_esp_reass_success_four_frags_ooo(odp_ipsec_sa_t out_sa, @@ -1869,7 +1875,8 @@ static void test_in_ipv4_esp_reass_success_four_frags_ooo(odp_ipsec_sa_t out_sa, test_multi_out_in(out_sa, in_sa, ODPH_IPV4, ARRAY_SIZE(input_packets), input_packets, - result_packet); + result_packet, + ODP_PACKET_REASS_COMPLETE); } static void test_in_ipv4_esp_reass_incomp_missing(odp_ipsec_sa_t out_sa, @@ -1883,7 +1890,8 @@ static void test_in_ipv4_esp_reass_incomp_missing(odp_ipsec_sa_t out_sa, test_multi_out_in(out_sa, in_sa, ODPH_IPV4, ARRAY_SIZE(input_packets), input_packets, - result_packet); + result_packet, + ODP_PACKET_REASS_INCOMPLETE); } static void test_in_ipv4_esp_reass_success(void) @@ -2001,7 +2009,8 @@ static void test_in_ipv6_esp_reass_success_two_frags(odp_ipsec_sa_t out_sa, test_multi_out_in(out_sa, in_sa, ODPH_IPV6, ARRAY_SIZE(input_packets), input_packets, - result_packet); + result_packet, + ODP_PACKET_REASS_COMPLETE); } static void test_in_ipv6_esp_reass_success_four_frags(odp_ipsec_sa_t out_sa, @@ -2018,7 +2027,8 @@ static void test_in_ipv6_esp_reass_success_four_frags(odp_ipsec_sa_t out_sa, test_multi_out_in(out_sa, in_sa, ODPH_IPV6, ARRAY_SIZE(input_packets), input_packets, - result_packet); + result_packet, + ODP_PACKET_REASS_COMPLETE); } static void test_in_ipv6_esp_reass_success_two_frags_ooo(odp_ipsec_sa_t out_sa, @@ -2033,7 +2043,8 @@ static void test_in_ipv6_esp_reass_success_two_frags_ooo(odp_ipsec_sa_t out_sa, test_multi_out_in(out_sa, in_sa, ODPH_IPV6, ARRAY_SIZE(input_packets), input_packets, - result_packet); + result_packet, + ODP_PACKET_REASS_COMPLETE); } static void test_in_ipv6_esp_reass_success_four_frags_ooo(odp_ipsec_sa_t out_sa, @@ -2050,7 +2061,8 @@ static void test_in_ipv6_esp_reass_success_four_frags_ooo(odp_ipsec_sa_t out_sa, test_multi_out_in(out_sa, in_sa, ODPH_IPV6, ARRAY_SIZE(input_packets), input_packets, - result_packet); + result_packet, + ODP_PACKET_REASS_COMPLETE); } static void test_in_ipv6_esp_reass_incomp_missing(odp_ipsec_sa_t out_sa, @@ -2064,7 +2076,8 @@ static void test_in_ipv6_esp_reass_incomp_missing(odp_ipsec_sa_t out_sa, test_multi_out_in(out_sa, in_sa, ODPH_IPV6, ARRAY_SIZE(input_packets), input_packets, - result_packet); + result_packet, + ODP_PACKET_REASS_INCOMPLETE); } static void test_in_ipv6_esp_reass_success(void) diff --git a/test/validation/api/ipsec/reass_test_vectors.c b/test/validation/api/ipsec/reass_test_vectors.c new file mode 100644 index 000000000..c3bb2bfd4 --- /dev/null +++ b/test/validation/api/ipsec/reass_test_vectors.c @@ -0,0 +1,353 @@ +/* Copyright (c) 2021, Marvell + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "ipsec.h" + +/* The source file includes below test vectors */ + +/* IPv6: + * + * 1) pkt_ipv6_udp_p1 + * pkt_ipv6_udp_p1_f1 + * pkt_ipv6_udp_p1_f2 + * + * 2) pkt_ipv6_udp_p2 + * pkt_ipv6_udp_p2_f1 + * pkt_ipv6_udp_p2_f2 + * pkt_ipv6_udp_p2_f3 + * pkt_ipv6_udp_p2_f4 + */ + +/* IPv4: + * + * 1) pkt_ipv4_udp_p1 + * pkt_ipv4_udp_p1_f1 + * pkt_ipv4_udp_p1_f2 + * + * 2) pkt_ipv4_udp_p2 + * pkt_ipv4_udp_p2_f1 + * pkt_ipv4_udp_p2_f2 + * pkt_ipv4_udp_p2_f3 + * pkt_ipv4_udp_p2_f4 + */ + +ipsec_test_packet pkt_ipv6_udp_p1 = { + .len = 1514, + .l2_offset = 0, + .l3_offset = 14, + .l4_offset = 54, + .data = { + /* ETH */ + 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, + 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x86, 0xdd, + + /* IP */ + 0x60, 0x00, 0x00, 0x00, 0x05, 0xb4, 0x11, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x0d, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x02, 0x00, 0x00, 0x02, + + /* UDP */ + 0x08, 0x00, 0x27, 0x10, 0x05, 0xb4, 0x2b, 0xe8, + }, +}; + +ipsec_test_packet pkt_ipv6_udp_p1_f1 = { + .len = 1398, + .l2_offset = 0, + .l3_offset = 14, + .l4_offset = 62, + .data = { + /* ETH */ + 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, + 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x86, 0xdd, + + /* IP */ + 0x60, 0x00, 0x00, 0x00, 0x05, 0x40, 0x2c, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x0d, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x02, 0x00, 0x00, 0x02, + 0x11, 0x00, 0x00, 0x01, 0x5c, 0x92, 0xac, 0xf1, + + /* UDP */ + 0x08, 0x00, 0x27, 0x10, 0x05, 0xb4, 0x2b, 0xe8, + }, +}; + +ipsec_test_packet pkt_ipv6_udp_p1_f2 = { + .len = 186, + .l2_offset = 0, + .l3_offset = 14, + .l4_offset = 62, + .data = { + /* ETH */ + 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, + 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x86, 0xdd, + + /* IP */ + 0x60, 0x00, 0x00, 0x00, 0x00, 0x84, 0x2c, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x0d, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x02, 0x00, 0x00, 0x02, + 0x11, 0x00, 0x05, 0x38, 0x5c, 0x92, 0xac, 0xf1, + }, +}; + +ipsec_test_packet pkt_ipv6_udp_p2 = { + .len = 4496, + .l2_offset = 0, + .l3_offset = 14, + .l4_offset = 54, + .data = { + /* ETH */ + 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, + 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x86, 0xdd, + + /* IP */ + 0x60, 0x00, 0x00, 0x00, 0x11, 0x5a, 0x11, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x0d, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x02, 0x00, 0x00, 0x02, + + /* UDP */ + 0x08, 0x00, 0x27, 0x10, 0x11, 0x5a, 0x8a, 0x11, + }, +}; + +ipsec_test_packet pkt_ipv6_udp_p2_f1 = { + .len = 1398, + .l2_offset = 0, + .l3_offset = 14, + .l4_offset = 62, + .data = { + /* ETH */ + 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, + 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x86, 0xdd, + + /* IP */ + 0x60, 0x00, 0x00, 0x00, 0x05, 0x40, 0x2c, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x0d, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x02, 0x00, 0x00, 0x02, + 0x11, 0x00, 0x00, 0x01, 0x64, 0x6c, 0x68, 0x9f, + + /* UDP */ + 0x08, 0x00, 0x27, 0x10, 0x11, 0x5a, 0x8a, 0x11, + }, +}; + +ipsec_test_packet pkt_ipv6_udp_p2_f2 = { + .len = 1398, + .l2_offset = 0, + .l3_offset = 14, + .l4_offset = 62, + .data = { + /* ETH */ + 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, + 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x86, 0xdd, + + /* IP */ + 0x60, 0x00, 0x00, 0x00, 0x05, 0x40, 0x2c, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x0d, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x02, 0x00, 0x00, 0x02, + 0x11, 0x00, 0x05, 0x39, 0x64, 0x6c, 0x68, 0x9f, + }, +}; + +ipsec_test_packet pkt_ipv6_udp_p2_f3 = { + .len = 1398, + .l2_offset = 0, + .l3_offset = 14, + .l4_offset = 62, + .data = { + /* ETH */ + 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, + 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x86, 0xdd, + + /* IP */ + 0x60, 0x00, 0x00, 0x00, 0x05, 0x40, 0x2c, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x0d, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x02, 0x00, 0x00, 0x02, + 0x11, 0x00, 0x0a, 0x71, 0x64, 0x6c, 0x68, 0x9f, + }, +}; + +ipsec_test_packet pkt_ipv6_udp_p2_f4 = { + .len = 496, + .l2_offset = 0, + .l3_offset = 14, + .l4_offset = 62, + .data = { + /* ETH */ + 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, + 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x86, 0xdd, + + /* IP */ + 0x60, 0x00, 0x00, 0x00, 0x01, 0xba, 0x2c, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x0d, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x02, 0x00, 0x00, 0x02, + 0x11, 0x00, 0x0f, 0xa8, 0x64, 0x6c, 0x68, 0x9f, + }, +}; + +ipsec_test_packet pkt_ipv4_udp_p1 = { + .len = 1514, + .l2_offset = 0, + .l3_offset = 14, + .l4_offset = 34, + .data = { + /* ETH */ + 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, + 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00, + + /* IP */ + 0x45, 0x00, 0x05, 0xdc, 0x00, 0x01, 0x00, 0x00, + 0x40, 0x11, 0x66, 0x0d, 0x0d, 0x00, 0x00, 0x02, + 0x02, 0x00, 0x00, 0x02, + + /* UDP */ + 0x08, 0x00, 0x27, 0x10, 0x05, 0xc8, 0xb8, 0x4c, + }, +}; + +ipsec_test_packet pkt_ipv4_udp_p1_f1 = { + .len = 1434, + .l2_offset = 0, + .l3_offset = 14, + .l4_offset = 34, + .data = { + /* ETH */ + 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, + 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00, + + /* IP */ + 0x45, 0x00, 0x05, 0x8c, 0x00, 0x01, 0x20, 0x00, + 0x40, 0x11, 0x46, 0x5d, 0x0d, 0x00, 0x00, 0x02, + 0x02, 0x00, 0x00, 0x02, + + /* UDP */ + 0x08, 0x00, 0x27, 0x10, 0x05, 0xc8, 0xb8, 0x4c, + }, +}; + +ipsec_test_packet pkt_ipv4_udp_p1_f2 = { + .len = 114, + .l2_offset = 0, + .l3_offset = 14, + .l4_offset = 34, + .data = { + /* ETH */ + 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, + 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00, + + /* IP */ + 0x45, 0x00, 0x00, 0x64, 0x00, 0x01, 0x00, 0xaf, + 0x40, 0x11, 0x6a, 0xd6, 0x0d, 0x00, 0x00, 0x02, + 0x02, 0x00, 0x00, 0x02, + }, +}; + +ipsec_test_packet pkt_ipv4_udp_p2 = { + .len = 4496, + .l2_offset = 0, + .l3_offset = 14, + .l4_offset = 34, + .data = { + /* ETH */ + 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, + 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00, + + /* IP */ + 0x45, 0x00, 0x11, 0x82, 0x00, 0x02, 0x00, 0x00, + 0x40, 0x11, 0x5a, 0x66, 0x0d, 0x00, 0x00, 0x02, + 0x02, 0x00, 0x00, 0x02, + + /* UDP */ + 0x08, 0x00, 0x27, 0x10, 0x11, 0x6e, 0x16, 0x76, + }, +}; + +ipsec_test_packet pkt_ipv4_udp_p2_f1 = { + .len = 1434, + .l2_offset = 0, + .l3_offset = 14, + .l4_offset = 34, + .data = { + /* ETH */ + 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, + 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00, + + /* IP */ + 0x45, 0x00, 0x05, 0x8c, 0x00, 0x02, 0x20, 0x00, + 0x40, 0x11, 0x46, 0x5c, 0x0d, 0x00, 0x00, 0x02, + 0x02, 0x00, 0x00, 0x02, + + /* UDP */ + 0x08, 0x00, 0x27, 0x10, 0x11, 0x6e, 0x16, 0x76, + }, +}; + +ipsec_test_packet pkt_ipv4_udp_p2_f2 = { + .len = 1434, + .l2_offset = 0, + .l3_offset = 14, + .l4_offset = 34, + .data = { + /* ETH */ + 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, + 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00, + + /* IP */ + 0x45, 0x00, 0x05, 0x8c, 0x00, 0x02, 0x20, 0xaf, + 0x40, 0x11, 0x45, 0xad, 0x0d, 0x00, 0x00, 0x02, + 0x02, 0x00, 0x00, 0x02, + }, +}; + +ipsec_test_packet pkt_ipv4_udp_p2_f3 = { + .len = 1434, + .l2_offset = 0, + .l3_offset = 14, + .l4_offset = 34, + .data = { + /* ETH */ + 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, + 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00, + + /* IP */ + 0x45, 0x00, 0x05, 0x8c, 0x00, 0x02, 0x21, 0x5e, + 0x40, 0x11, 0x44, 0xfe, 0x0d, 0x00, 0x00, 0x02, + 0x02, 0x00, 0x00, 0x02, + }, +}; + +ipsec_test_packet pkt_ipv4_udp_p2_f4 = { + .len = 296, + .l2_offset = 0, + .l3_offset = 14, + .l4_offset = 34, + .data = { + /* ETH */ + 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, + 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00, + + /* IP */ + 0x45, 0x00, 0x01, 0x1a, 0x00, 0x02, 0x02, 0x0d, + 0x40, 0x11, 0x68, 0xc1, 0x0d, 0x00, 0x00, 0x02, + 0x02, 0x00, 0x00, 0x02, + }, +}; diff --git a/test/validation/api/ipsec/reass_test_vectors.h b/test/validation/api/ipsec/reass_test_vectors.h index 4fbb1ebaf..02b41c573 100644 --- a/test/validation/api/ipsec/reass_test_vectors.h +++ b/test/validation/api/ipsec/reass_test_vectors.h @@ -7,351 +7,23 @@ #ifndef _ODP_REASS_TEST_VECTORS_H_ #define _ODP_REASS_TEST_VECTORS_H_ -/* The header file includes below test vectors */ - -/* IPv6: - * - * 1) pkt_ipv6_udp_p1 - * pkt_ipv6_udp_p1_f1 - * pkt_ipv6_udp_p1_f2 - * - * 2) pkt_ipv6_udp_p2 - * pkt_ipv6_udp_p2_f1 - * pkt_ipv6_udp_p2_f2 - * pkt_ipv6_udp_p2_f3 - * pkt_ipv6_udp_p2_f4 - */ - -/* IPv4: - * - * 1) pkt_ipv4_udp_p1 - * pkt_ipv4_udp_p1_f1 - * pkt_ipv4_udp_p1_f2 - * - * 2) pkt_ipv4_udp_p2 - * pkt_ipv4_udp_p2_f1 - * pkt_ipv4_udp_p2_f2 - * pkt_ipv4_udp_p2_f3 - * pkt_ipv4_udp_p2_f4 - */ - -static ipsec_test_packet pkt_ipv6_udp_p1 = { - .len = 1514, - .l2_offset = 0, - .l3_offset = 14, - .l4_offset = 54, - .data = { - /* ETH */ - 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, - 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x86, 0xdd, - - /* IP */ - 0x60, 0x00, 0x00, 0x00, 0x05, 0xb4, 0x11, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x0d, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x02, 0x00, 0x00, 0x02, - - /* UDP */ - 0x08, 0x00, 0x27, 0x10, 0x05, 0xb4, 0x2b, 0xe8, - }, -}; - -static ipsec_test_packet pkt_ipv6_udp_p1_f1 = { - .len = 1398, - .l2_offset = 0, - .l3_offset = 14, - .l4_offset = 62, - .data = { - /* ETH */ - 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, - 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x86, 0xdd, - - /* IP */ - 0x60, 0x00, 0x00, 0x00, 0x05, 0x40, 0x2c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x0d, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x02, 0x00, 0x00, 0x02, - 0x11, 0x00, 0x00, 0x01, 0x5c, 0x92, 0xac, 0xf1, - - /* UDP */ - 0x08, 0x00, 0x27, 0x10, 0x05, 0xb4, 0x2b, 0xe8, - }, -}; - -static ipsec_test_packet pkt_ipv6_udp_p1_f2 = { - .len = 186, - .l2_offset = 0, - .l3_offset = 14, - .l4_offset = 62, - .data = { - /* ETH */ - 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, - 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x86, 0xdd, - - /* IP */ - 0x60, 0x00, 0x00, 0x00, 0x00, 0x84, 0x2c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x0d, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x02, 0x00, 0x00, 0x02, - 0x11, 0x00, 0x05, 0x38, 0x5c, 0x92, 0xac, 0xf1, - }, -}; - -static ipsec_test_packet pkt_ipv6_udp_p2 = { - .len = 4496, - .l2_offset = 0, - .l3_offset = 14, - .l4_offset = 54, - .data = { - /* ETH */ - 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, - 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x86, 0xdd, - - /* IP */ - 0x60, 0x00, 0x00, 0x00, 0x11, 0x5a, 0x11, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x0d, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x02, 0x00, 0x00, 0x02, - - /* UDP */ - 0x08, 0x00, 0x27, 0x10, 0x11, 0x5a, 0x8a, 0x11, - }, -}; - -static ipsec_test_packet pkt_ipv6_udp_p2_f1 = { - .len = 1398, - .l2_offset = 0, - .l3_offset = 14, - .l4_offset = 62, - .data = { - /* ETH */ - 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, - 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x86, 0xdd, - - /* IP */ - 0x60, 0x00, 0x00, 0x00, 0x05, 0x40, 0x2c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x0d, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x02, 0x00, 0x00, 0x02, - 0x11, 0x00, 0x00, 0x01, 0x64, 0x6c, 0x68, 0x9f, - - /* UDP */ - 0x08, 0x00, 0x27, 0x10, 0x11, 0x5a, 0x8a, 0x11, - }, -}; - -static ipsec_test_packet pkt_ipv6_udp_p2_f2 = { - .len = 1398, - .l2_offset = 0, - .l3_offset = 14, - .l4_offset = 62, - .data = { - /* ETH */ - 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, - 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x86, 0xdd, - - /* IP */ - 0x60, 0x00, 0x00, 0x00, 0x05, 0x40, 0x2c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x0d, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x02, 0x00, 0x00, 0x02, - 0x11, 0x00, 0x05, 0x39, 0x64, 0x6c, 0x68, 0x9f, - }, -}; - -static ipsec_test_packet pkt_ipv6_udp_p2_f3 = { - .len = 1398, - .l2_offset = 0, - .l3_offset = 14, - .l4_offset = 62, - .data = { - /* ETH */ - 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, - 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x86, 0xdd, - - /* IP */ - 0x60, 0x00, 0x00, 0x00, 0x05, 0x40, 0x2c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x0d, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x02, 0x00, 0x00, 0x02, - 0x11, 0x00, 0x0a, 0x71, 0x64, 0x6c, 0x68, 0x9f, - }, -}; - -static ipsec_test_packet pkt_ipv6_udp_p2_f4 = { - .len = 496, - .l2_offset = 0, - .l3_offset = 14, - .l4_offset = 62, - .data = { - /* ETH */ - 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, - 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x86, 0xdd, - - /* IP */ - 0x60, 0x00, 0x00, 0x00, 0x01, 0xba, 0x2c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x0d, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x02, 0x00, 0x00, 0x02, - 0x11, 0x00, 0x0f, 0xa8, 0x64, 0x6c, 0x68, 0x9f, - }, -}; - -static ipsec_test_packet pkt_ipv4_udp_p1 = { - .len = 1514, - .l2_offset = 0, - .l3_offset = 14, - .l4_offset = 34, - .data = { - /* ETH */ - 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, - 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00, - - /* IP */ - 0x45, 0x00, 0x05, 0xdc, 0x00, 0x01, 0x00, 0x00, - 0x40, 0x11, 0x66, 0x0d, 0x0d, 0x00, 0x00, 0x02, - 0x02, 0x00, 0x00, 0x02, - - /* UDP */ - 0x08, 0x00, 0x27, 0x10, 0x05, 0xc8, 0xb8, 0x4c, - }, -}; - -static ipsec_test_packet pkt_ipv4_udp_p1_f1 = { - .len = 1434, - .l2_offset = 0, - .l3_offset = 14, - .l4_offset = 34, - .data = { - /* ETH */ - 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, - 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00, - - /* IP */ - 0x45, 0x00, 0x05, 0x8c, 0x00, 0x01, 0x20, 0x00, - 0x40, 0x11, 0x46, 0x5d, 0x0d, 0x00, 0x00, 0x02, - 0x02, 0x00, 0x00, 0x02, - - /* UDP */ - 0x08, 0x00, 0x27, 0x10, 0x05, 0xc8, 0xb8, 0x4c, - }, -}; - -static ipsec_test_packet pkt_ipv4_udp_p1_f2 = { - .len = 114, - .l2_offset = 0, - .l3_offset = 14, - .l4_offset = 34, - .data = { - /* ETH */ - 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, - 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00, - - /* IP */ - 0x45, 0x00, 0x00, 0x64, 0x00, 0x01, 0x00, 0xaf, - 0x40, 0x11, 0x6a, 0xd6, 0x0d, 0x00, 0x00, 0x02, - 0x02, 0x00, 0x00, 0x02, - }, -}; - -static ipsec_test_packet pkt_ipv4_udp_p2 = { - .len = 4496, - .l2_offset = 0, - .l3_offset = 14, - .l4_offset = 34, - .data = { - /* ETH */ - 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, - 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00, - - /* IP */ - 0x45, 0x00, 0x11, 0x82, 0x00, 0x02, 0x00, 0x00, - 0x40, 0x11, 0x5a, 0x66, 0x0d, 0x00, 0x00, 0x02, - 0x02, 0x00, 0x00, 0x02, - - /* UDP */ - 0x08, 0x00, 0x27, 0x10, 0x11, 0x6e, 0x16, 0x76, - }, -}; - -static ipsec_test_packet pkt_ipv4_udp_p2_f1 = { - .len = 1434, - .l2_offset = 0, - .l3_offset = 14, - .l4_offset = 34, - .data = { - /* ETH */ - 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, - 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00, - - /* IP */ - 0x45, 0x00, 0x05, 0x8c, 0x00, 0x02, 0x20, 0x00, - 0x40, 0x11, 0x46, 0x5c, 0x0d, 0x00, 0x00, 0x02, - 0x02, 0x00, 0x00, 0x02, - - /* UDP */ - 0x08, 0x00, 0x27, 0x10, 0x11, 0x6e, 0x16, 0x76, - }, -}; - -static ipsec_test_packet pkt_ipv4_udp_p2_f2 = { - .len = 1434, - .l2_offset = 0, - .l3_offset = 14, - .l4_offset = 34, - .data = { - /* ETH */ - 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, - 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00, - - /* IP */ - 0x45, 0x00, 0x05, 0x8c, 0x00, 0x02, 0x20, 0xaf, - 0x40, 0x11, 0x45, 0xad, 0x0d, 0x00, 0x00, 0x02, - 0x02, 0x00, 0x00, 0x02, - }, -}; - -static ipsec_test_packet pkt_ipv4_udp_p2_f3 = { - .len = 1434, - .l2_offset = 0, - .l3_offset = 14, - .l4_offset = 34, - .data = { - /* ETH */ - 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, - 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00, - - /* IP */ - 0x45, 0x00, 0x05, 0x8c, 0x00, 0x02, 0x21, 0x5e, - 0x40, 0x11, 0x44, 0xfe, 0x0d, 0x00, 0x00, 0x02, - 0x02, 0x00, 0x00, 0x02, - }, -}; - -static ipsec_test_packet pkt_ipv4_udp_p2_f4 = { - .len = 296, - .l2_offset = 0, - .l3_offset = 14, - .l4_offset = 34, - .data = { - /* ETH */ - 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, - 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00, - - /* IP */ - 0x45, 0x00, 0x01, 0x1a, 0x00, 0x02, 0x02, 0x0d, - 0x40, 0x11, 0x68, 0xc1, 0x0d, 0x00, 0x00, 0x02, - 0x02, 0x00, 0x00, 0x02, - }, -}; +extern ipsec_test_packet pkt_ipv6_udp_p1; +extern ipsec_test_packet pkt_ipv6_udp_p1_f1; +extern ipsec_test_packet pkt_ipv6_udp_p1_f2; +extern ipsec_test_packet pkt_ipv6_udp_p2; +extern ipsec_test_packet pkt_ipv6_udp_p2_f1; +extern ipsec_test_packet pkt_ipv6_udp_p2_f2; +extern ipsec_test_packet pkt_ipv6_udp_p2_f3; +extern ipsec_test_packet pkt_ipv6_udp_p2_f4; + +extern ipsec_test_packet pkt_ipv4_udp_p1; +extern ipsec_test_packet pkt_ipv4_udp_p1_f1; +extern ipsec_test_packet pkt_ipv4_udp_p1_f2; +extern ipsec_test_packet pkt_ipv4_udp_p2; +extern ipsec_test_packet pkt_ipv4_udp_p2_f1; +extern ipsec_test_packet pkt_ipv4_udp_p2_f2; +extern ipsec_test_packet pkt_ipv4_udp_p2_f3; +extern ipsec_test_packet pkt_ipv4_udp_p2_f4; static inline void test_vector_payload_populate(ipsec_test_packet *pkt, odp_bool_t first_frag) diff --git a/test/validation/api/pool/pool.c b/test/validation/api/pool/pool.c index d791063e2..861ac18ae 100644 --- a/test/validation/api/pool/pool.c +++ b/test/validation/api/pool/pool.c @@ -893,6 +893,63 @@ static void pool_test_pool_index(void) } } +static void pool_test_create_max_pkt_pools(void) +{ + uint32_t max_pools = global_pool_capa.pkt.max_pools; + uint32_t i, num_pools, num_shm; + odp_pool_t pool[max_pools]; + odp_pool_param_t param; + odp_shm_capability_t shm_capa; + uint32_t shm_size = 32; + uint32_t uarea_size = 32; + + CU_ASSERT_FATAL(max_pools > 0); + + /* Reserve maximum number of SHM blocks */ + CU_ASSERT_FATAL(odp_shm_capability(&shm_capa) == 0); + CU_ASSERT_FATAL(shm_capa.max_blocks > 0); + + odp_shm_t shm[shm_capa.max_blocks]; + + if (shm_capa.max_size && shm_capa.max_size < shm_size) + shm_size = shm_capa.max_size; + + for (i = 0; i < shm_capa.max_blocks; i++) { + shm[i] = odp_shm_reserve(NULL, shm_size, 0, 0); + + if (shm[i] == ODP_SHM_INVALID) + break; + } + num_shm = i; + CU_ASSERT(num_shm == shm_capa.max_blocks); + + /* Create maximum number of packet pools */ + if (global_pool_capa.pkt.max_uarea_size && global_pool_capa.pkt.max_uarea_size < uarea_size) + uarea_size = global_pool_capa.pkt.max_uarea_size; + + odp_pool_param_init(¶m); + param.type = ODP_POOL_PACKET; + param.pkt.len = PKT_LEN; + param.pkt.num = 1; + param.pkt.max_num = 1; + param.pkt.uarea_size = uarea_size; + + for (i = 0; i < max_pools; i++) { + pool[i] = odp_pool_create(NULL, ¶m); + + if (pool[i] == ODP_POOL_INVALID) + break; + } + num_pools = i; + CU_ASSERT(num_pools == max_pools); + + for (i = 0; i < num_pools; i++) + CU_ASSERT(odp_pool_destroy(pool[i]) == 0); + + for (i = 0; i < num_shm; i++) + CU_ASSERT(odp_shm_free(shm[i]) == 0); +} + static int pool_check_buffer_pool_statistics(void) { if (global_pool_capa.buf.stats.all == 0) @@ -1706,6 +1763,7 @@ odp_testinfo_t pool_suite[] = { ODP_TEST_INFO(pool_test_tmo_max_num), ODP_TEST_INFO(pool_test_create_after_fork), ODP_TEST_INFO(pool_test_pool_index), + ODP_TEST_INFO(pool_test_create_max_pkt_pools), ODP_TEST_INFO_CONDITIONAL(pool_test_buffer_pool_statistics, pool_check_buffer_pool_statistics), ODP_TEST_INFO_CONDITIONAL(pool_test_packet_pool_statistics, |