example: ipsec: exit workers and destroy resources

Modified worker main loop to exit when the main thread
commands. Added destroy and terminate calls to clean up
resources before exiting. For example, ODP linux-generic
implementation does not free shm memory without proper
destroy and terminate calls.

Signed-off-by: Petri Savolainen <petri.savolainen@nokia.com>
Reviewed-by: Dmitry Eremin-Solenikov <deremin-solenikov@cavium.com>

3 files changed, 63 insertions, 17 deletions

diff --git a/example/ipsec/odp_ipsec.c b/example/ipsec/odp_ipsec.c
index 10bd06e0b..96c0fe613 100644
--- a/example/ipsec/odp_ipsec.c
+++ b/example/ipsec/odp_ipsec.c
@@ -98,6 +98,7 @@ typedef struct {
 	odp_barrier_t sync_barrier;
 	odp_queue_t poll_queues[MAX_POLL_QUEUES];
 	int num_polled_queues;
+	volatile int stop_workers;
 } global_data_t;
 
 /* helper funcs */
@@ -279,7 +280,7 @@ odp_queue_t polled_odp_queue_create(const char *name,
 static inline
 odp_event_t odp_schedule_cb(odp_queue_t *from)
 {
-	return odp_schedule(from, ODP_SCHED_WAIT);
+	return odp_schedule(from, ODP_SCHED_NO_WAIT);
 }
 
 /**
@@ -290,18 +291,15 @@ odp_event_t polled_odp_schedule_cb(odp_queue_t *from)
 {
 	int idx = 0;
 
-	while (1) {
-		if (idx >= global->num_polled_queues)
-			idx = 0;
-
+	while (idx < global->num_polled_queues) {
 		odp_queue_t queue = global->poll_queues[idx++];
-		odp_event_t buf;
+		odp_event_t ev;
 
-		buf = odp_queue_deq(queue);
+		ev = odp_queue_deq(queue);
 
-		if (ODP_EVENT_INVALID != buf) {
+		if (ODP_EVENT_INVALID != ev) {
 			*from = queue;
-			return buf;
+			return ev;
 		}
 	}
 
@@ -1057,7 +1055,7 @@ int pktio_thread(void *arg EXAMPLE_UNUSED)
 	odp_barrier_wait(&global->sync_barrier);
 
 	/* Loop packets */
-	for (;;) {
+	while (global->stop_workers == 0) {
 		pkt_disposition_e rc;
 		pkt_ctx_t   *ctx;
 		odp_queue_t  dispatchq;
@@ -1066,10 +1064,8 @@ int pktio_thread(void *arg EXAMPLE_UNUSED)
 		/* Use schedule to get event from any input queue */
 		ev = schedule(&dispatchq);
 
-		if (ev == ODP_EVENT_INVALID) {
-			EXAMPLE_ERR("Error: Bad event handle\n");
-			exit(EXIT_FAILURE);
-		}
+		if (ev == ODP_EVENT_INVALID)
+			continue;
 
 		/* Determine new work versus completion or sequence number */
 		if (ODP_EVENT_PACKET == odp_event_types(ev, &subtype)) {
@@ -1191,7 +1187,6 @@ int pktio_thread(void *arg EXAMPLE_UNUSED)
 		}
 	}
 
-	/* unreachable */
 	return 0;
 }
 
@@ -1338,6 +1333,7 @@ main(int argc, char *argv[])
 	/*
 	 * Create and init worker threads
 	 */
+	memset(thread_tbl, 0, sizeof(thread_tbl));
 	memset(&thr_params, 0, sizeof(thr_params));
 	thr_params.start    = pktio_thread;
 	thr_params.arg      = NULL;
@@ -1356,10 +1352,13 @@ main(int argc, char *argv[])
 			sleep(1);
 		} while (!done);
 		printf("All received\n");
-	} else {
-		odph_odpthreads_join(thread_tbl);
 	}
 
+	global->stop_workers = 1;
+	odp_mb_full();
+
+	odph_odpthreads_join(thread_tbl);
+
 	/* Stop and close used pktio devices */
 	for (i = 0; i < global->appl.if_count; i++) {
 		odp_pktio_t pktio = odp_pktio_lookup(global->appl.if_names[i]);
@@ -1377,6 +1376,21 @@ main(int argc, char *argv[])
 	free(global->appl.if_names);
 	free(global->appl.if_str);
 
+	if (destroy_ipsec_cache())
+		EXAMPLE_ERR("Error: crypto session destroy failed\n");
+
+	if (odp_queue_destroy(global->completionq))
+		EXAMPLE_ERR("Error: queue destroy failed\n");
+	if (odp_queue_destroy(global->seqnumq))
+		EXAMPLE_ERR("Error: queue destroy failed\n");
+
+	if (odp_pool_destroy(global->pkt_pool))
+		EXAMPLE_ERR("Error: pool destroy failed\n");
+	if (odp_pool_destroy(global->ctx_pool))
+		EXAMPLE_ERR("Error: pool destroy failed\n");
+	if (odp_pool_destroy(global->out_pool))
+		EXAMPLE_ERR("Error: pool destroy failed\n");
+
 	shm = odp_shm_lookup("shm_ipsec_cache");
 	if (odp_shm_free(shm) != 0)
 		EXAMPLE_ERR("Error: shm free shm_ipsec_cache failed\n");
@@ -1400,6 +1414,16 @@ main(int argc, char *argv[])
 		exit(EXIT_FAILURE);
 	}
 
+	if (odp_term_local()) {
+		EXAMPLE_ERR("Error: term local failed\n");
+		exit(EXIT_FAILURE);
+	}
+
+	if (odp_term_global(instance)) {
+		EXAMPLE_ERR("Error: term global failed\n");
+		exit(EXIT_FAILURE);
+	}
+
 	printf("Exit\n\n");
 
 	return 0;
diff --git a/example/ipsec/odp_ipsec_cache.c b/example/ipsec/odp_ipsec_cache.c
index 5bbc497cf..362e64a44 100644
--- a/example/ipsec/odp_ipsec_cache.c
+++ b/example/ipsec/odp_ipsec_cache.c
@@ -21,6 +21,7 @@ ipsec_cache_t *ipsec_cache;
 void init_ipsec_cache(void)
 {
 	odp_shm_t shm;
+	int i;
 
 	shm = odp_shm_reserve("shm_ipsec_cache",
 			      sizeof(ipsec_cache_t),
@@ -38,6 +39,10 @@ void init_ipsec_cache(void)
 		exit(EXIT_FAILURE);
 	}
 	memset(ipsec_cache, 0, sizeof(*ipsec_cache));
+
+	for (i = 0; i < MAX_DB; i++)
+		ipsec_cache->array[i].state.session =
+		ODP_CRYPTO_SESSION_INVALID;
 }
 
 int create_ipsec_cache_entry(sa_db_entry_t *cipher_sa,
@@ -223,3 +228,18 @@ ipsec_cache_entry_t *find_ipsec_cache_entry_out(uint32_t src_ip,
 	}
 	return entry;
 }
+
+int destroy_ipsec_cache(void)
+{
+	ipsec_cache_entry_t *entry;
+	int i;
+	int ret = 0;
+
+	for (i = 0; i < MAX_DB; i++) {
+		entry = &ipsec_cache->array[i];
+		if (entry->state.session != ODP_CRYPTO_SESSION_INVALID)
+			ret += odp_crypto_session_destroy(entry->state.session);
+	}
+
+	return ret;
+}
diff --git a/example/ipsec/odp_ipsec_cache.h b/example/ipsec/odp_ipsec_cache.h
index fb2416823..1523778ff 100644
--- a/example/ipsec/odp_ipsec_cache.h
+++ b/example/ipsec/odp_ipsec_cache.h
@@ -127,6 +127,8 @@ ipsec_cache_entry_t *find_ipsec_cache_entry_out(uint32_t src_ip,
 						uint32_t dst_ip,
 						uint8_t proto);
 
+int destroy_ipsec_cache(void);
+
 #ifdef __cplusplus
 }
 #endif