test: cunit: fix cunit asserts for multithreaded test apps

CUnit is not thread-safe and not aware of process mode, which causes
problems when CUnit assertions are used in multithreaded tests:
- Updating the recorded assert failures has data races
- Assertion failures in other than the main process are lost
- The longjmp performed at fatal assertion failure tries to jump to
  a stack frame of another thread, which is doomed to fail badly

Fix the problems by wrapping the commonly used CUnit assertions with
ODP macros and an ODP function that calls CUnit only if running in the
main thread and the main process. In other threads/processes record
failed assertions in shared memory and re-play them to CUnit within
the main thread/process when becoming single threaded again in
odp_cunit_thread_join().

This does not solve the general issue that careless use of fatal
assertions in the middle of tests may leave the test app in a state
from which it cannot recover.

Signed-off-by: Janne Peltonen <janne.peltonen@nokia.com>
Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com>

2 files changed, 177 insertions, 2 deletions

diff --git a/test/common/odp_cunit_common.c b/test/common/odp_cunit_common.c
index ebde4710a..935ac9f6d 100644
--- a/test/common/odp_cunit_common.c
+++ b/test/common/odp_cunit_common.c
@@ -6,9 +6,12 @@
  * SPDX-License-Identifier:     BSD-3-Clause
  */
 
+#define _GNU_SOURCE
+
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#include <sys/mman.h>
 #include <odp_api.h>
 #include "odp_cunit_common.h"
 #include <odp/helper/odph_api.h>
@@ -32,6 +35,7 @@ static odph_thread_t thread_tbl[MAX_WORKERS];
 static int threads_running;
 static odp_instance_t instance;
 static char *progname;
+static int (*thread_func)(void *);
 
 /*
  * global init/term functions which may be registered
@@ -46,6 +50,136 @@ static struct {
 
 static odp_suiteinfo_t *global_testsuites;
 
+#define MAX_STR_LEN 256
+#define MAX_FAILURES 10
+
+/* Recorded assertion failure for later CUnit call in the initial thread */
+typedef struct assertion_failure_t {
+	char cond[MAX_STR_LEN];
+	char file[MAX_STR_LEN];
+	unsigned int line;
+	int fatal;
+} assertion_failure_t;
+
+typedef struct thr_global_t {
+	assertion_failure_t failure[MAX_FAILURES];
+	unsigned long num_failures;
+} thr_global_t;
+
+static thr_global_t *thr_global;
+
+static __thread int initial_thread = 1; /* Are we the initial thread? */
+static __thread jmp_buf longjmp_env;
+
+void odp_cu_assert(CU_BOOL value, unsigned int line,
+		   const char *condition, const char *file, CU_BOOL fatal)
+{
+	unsigned long idx;
+
+	if (initial_thread) {
+		CU_assertImplementation(value, line, condition, file, "", fatal);
+		return;
+	}
+
+	/* Assertion ok, just return */
+	if (value)
+		return;
+
+	/*
+	 * Non-initial thread/process cannot call CUnit assert because:
+	 *
+	 * - CU_assertImplementation() is not thread-safe
+	 * - In process mode an assertion failure would be lost because it
+	 *   would not be recorded in the memory of the initial process.
+	 * - Fatal asserts in CUnit perform longjmp which cannot be done in
+	 *   an other thread or process that did the setjmp.
+	 *
+	 * --> Record the assertion failure in shared memory so that it can be
+	 *     processed later in the context of the initial thread/process.
+	 * --> In fatal assert, longjmp within the current thread.
+	 */
+
+	idx = __atomic_fetch_add(&thr_global->num_failures, 1, __ATOMIC_RELAXED);
+
+	if (idx < MAX_FAILURES) {
+		assertion_failure_t *a = &thr_global->failure[idx];
+
+		strncpy(a->cond, condition, sizeof(a->cond));
+		strncpy(a->file, file, sizeof(a->file));
+		a->cond[sizeof(a->cond) - 1] = 0;
+		a->file[sizeof(a->file) - 1] = 0;
+		a->line = line;
+		a->fatal = fatal;
+	}
+
+	if (fatal)
+		longjmp(longjmp_env, 1);
+}
+
+static void handle_postponed_asserts(void)
+{
+	unsigned long num = thr_global->num_failures;
+
+	if (num > MAX_FAILURES)
+		num = MAX_FAILURES;
+
+	for (unsigned long n = 0; n < num; n++) {
+		assertion_failure_t *a = &thr_global->failure[n];
+
+		/*
+		 * Turn fatal failures into non-fatal failures as we are just
+		 * reporting them. Threads that saw fatal failures which
+		 * prevented them from continuing have already been stopped.
+		 */
+		CU_assertImplementation(0, a->line, a->cond, a->file, "", CU_FALSE);
+	}
+	thr_global->num_failures = 0;
+}
+
+static int threads_init(void)
+{
+	static int initialized;
+
+	if (initialized)
+		return 0;
+
+	/*
+	 * Use shared memory mapping for the global structure to make it
+	 * visible in the child processes if running in process mode.
+	 */
+	thr_global = mmap(NULL, sizeof(thr_global_t),
+			  PROT_READ | PROT_WRITE,
+			  MAP_SHARED | MAP_ANONYMOUS,
+			  -1, 0);
+	if (thr_global == MAP_FAILED)
+		return -1;
+
+	initialized = 1;
+	return 0;
+}
+
+static int run_thread(void *arg)
+{
+	int rc;
+
+	/* Make sure this is zero also in process mode "threads" */
+	initial_thread = 0;
+
+	if (setjmp(longjmp_env) == 0) {
+		/* Normal return, proceed to the thread function. */
+		rc = (*thread_func)(arg);
+	} else {
+		/*
+		 * Return from longjmp done by the thread function.
+		 * We return 0 here since odph_thread_join() does not like
+		 * nonzero exit statuses.
+		 */
+		rc = 0;
+	}
+
+	return rc;
+}
+
 int odp_cunit_thread_create(int num, int func_ptr(void *), void *const arg[], int priv)
 {
 	int i, ret;
@@ -59,6 +193,8 @@ int odp_cunit_thread_create(int num, int func_ptr(void *), void *const arg[], in
 		return -1;
 	}
 
+	thread_func = func_ptr;
+
 	odph_thread_common_param_init(&thr_common);
 
 	if (arg == NULL)
@@ -67,7 +203,7 @@ int odp_cunit_thread_create(int num, int func_ptr(void *), void *const arg[], in
 	for (i = 0; i < num; i++) {
 		odph_thread_param_init(&thr_param[i]);
 
-		thr_param[i].start    = func_ptr;
+		thr_param[i].start    = run_thread;
 		thr_param[i].thr_type = ODP_THREAD_WORKER;
 
 		if (arg)
@@ -104,6 +240,9 @@ int odp_cunit_thread_join(int num)
 		return -1;
 	}
 	threads_running = 0;
+	thread_func = 0;
+
+	handle_postponed_asserts();
 
 	return 0;
 }
@@ -518,6 +657,9 @@ int odp_cunit_update(odp_suiteinfo_t testsuites[])
  */
 int odp_cunit_register(odp_suiteinfo_t testsuites[])
 {
+	if (threads_init())
+		return -1;
+
 	/* call test executable init hook, if any */
 	if (global_init_term.global_init_ptr) {
 		if ((*global_init_term.global_init_ptr)(&instance) == 0) {
diff --git a/test/common/odp_cunit_common.h b/test/common/odp_cunit_common.h
index 80e434d23..cb3fe4b89 100644
--- a/test/common/odp_cunit_common.h
+++ b/test/common/odp_cunit_common.h
@@ -110,6 +110,9 @@ int odp_cunit_set_inactive(void);
 /* Check from CI_SKIP environment variable if the test case should be skipped by CI */
 int odp_cunit_ci_skip(const char *test_name);
 
+void odp_cu_assert(CU_BOOL value, unsigned int line,
+		   const char *condition, const char *file, CU_BOOL fatal);
+
 /*
  * Wrapper for CU_assertImplementation for the fatal asserts to show the
  * compiler and static analyzers that the function does not return if the
@@ -119,7 +122,7 @@ int odp_cunit_ci_skip(const char *test_name);
 static inline void odp_cu_assert_fatal(CU_BOOL value, unsigned int line,
 				       const char *condition, const char *file)
 {
-	CU_assertImplementation(value, line, condition, file, "", CU_TRUE);
+	odp_cu_assert(value, line, condition, file, CU_TRUE);
 
 	if (!value) {
 		/* not reached */
@@ -134,31 +137,61 @@ static inline void odp_cu_assert_fatal(CU_BOOL value, unsigned int line,
  * compatibility with CU and existing code that assumes this kind of macros.
  */
 
+#undef CU_ASSERT
+#define CU_ASSERT(value) \
+	{ odp_cu_assert((value), __LINE__, #value, __FILE__, CU_FALSE); }
+
 #undef CU_ASSERT_FATAL
 #define CU_ASSERT_FATAL(value) \
 	{ odp_cu_assert_fatal((value), __LINE__, #value, __FILE__); }
 
+#undef CU_FAIL
+#define CU_FAIL(msg) \
+	{ odp_cu_assert(CU_FALSE, __LINE__, ("CU_FAIL(" #msg ")"), __FILE__, CU_FALSE); }
+
 #undef CU_FAIL_FATAL
 #define CU_FAIL_FATAL(msg) \
 	{ odp_cu_assert_fatal(CU_FALSE, __LINE__, ("CU_FAIL_FATAL(" #msg ")"), __FILE__); }
 
+#undef CU_ASSERT_EQUAL
+#define CU_ASSERT_EQUAL(actual, expected) \
+	{ odp_cu_assert(((actual) == (expected)), __LINE__, \
+			("CU_ASSERT_EQUAL(" #actual "," #expected ")"), \
+			__FILE__, CU_FALSE); }
+
 #undef CU_ASSERT_EQUAL_FATAL
 #define CU_ASSERT_EQUAL_FATAL(actual, expected) \
 	{ odp_cu_assert_fatal(((actual) == (expected)), __LINE__, \
 			      ("CU_ASSERT_EQUAL_FATAL(" #actual "," #expected ")"), \
 			      __FILE__); }
 
+#undef CU_ASSERT_NOT_EQUAL
+#define CU_ASSERT_NOT_EQUAL(actual, expected) \
+	{ odp_cu_assert(((actual) != (expected)), __LINE__, \
+			("CU_ASSERT_NOT_EQUAL(" #actual "," #expected ")"), \
+			__FILE__, CU_FALSE); }
+
 #undef CU_ASSERT_NOT_EQUAL_FATAL
 #define CU_ASSERT_NOT_EQUAL_FATAL(actual, expected) \
 	{ odp_cu_assert_fatal(((actual) != (expected)), __LINE__, \
 			      ("CU_ASSERT_NOT_EQUAL_FATAL(" #actual "," #expected ")"), \
 			      __FILE__); }
 
+#undef CU_ASSERT_PTR_NULL
+#define CU_ASSERT_PTR_NULL(value) \
+	{ odp_cu_assert((NULL == (const void *)(value)), __LINE__, \
+			("CU_ASSERT_PTR_NULL(" #value ")"), __FILE__, CU_FALSE); }
+
 #undef CU_ASSERT_PTR_NULL_FATAL
 #define CU_ASSERT_PTR_NULL_FATAL(value) \
 	{ odp_cu_assert_fatal((NULL == (const void *)(value)), __LINE__, \
 			      ("CU_ASSERT_PTR_NULL_FATAL(" #value ")"), __FILE__); }
 
+#undef CU_ASSERT_PTR_NOT_NULL
+#define CU_ASSERT_PTR_NOT_NULL(value) \
+	{ odp_cu_assert((NULL != (const void *)(value)), __LINE__, \
+			("CU_ASSERT_PTR_NOT_NULL_FATAL(" #value ")"), __FILE__, CU_FALSE); }
+
 #undef CU_ASSERT_PTR_NOT_NULL_FATAL
 #define CU_ASSERT_PTR_NOT_NULL_FATAL(value) \
 	{ odp_cu_assert_fatal((NULL != (const void *)(value)), __LINE__, \