Make tests output more concise and clearer

Also refactor the code that generates this output.

This is only a first step, subsequent patches will further improve
the output.

Here is a sample of the old output:

[cpu 0x0000] NOTICE:  Starting unittest 'Template - Single core test'
[cpu 0x0000] NOTICE:  Unittest 'Template - Single core test' complete. Result: Passed
[cpu 0x0000] NOTICE:  Starting unittest 'Template - Multi core test'
[cpu 0x0000] NOTICE:  Unittest 'Template - Multi core test' complete. Result: Passed
========== TEST REPORT ==========
 # Test suite 'Template':
- Single core test: Passed
- Multi core test: Passed
=================================
Tests Skipped : 0
Tests Passed  : 2
Tests Failed  : 0
Tests Crashed : 0
Total tests   : 2
=================================
[cpu 0x0000] NOTICE:  Exiting tests.

And now the new output:

[cpu 0x0000] --
[cpu 0x0000] Running test suite 'Template'
[cpu 0x0000] Description: Template test code
[cpu 0x0000]
[cpu 0x0000] > Executing 'Single core test'
[cpu 0x0000]   TEST COMPLETE                                                 Passed
[cpu 0x0000]
[cpu 0x0000] > Executing 'Multi core test'
[cpu 0x0000]   TEST COMPLETE                                                 Passed
[cpu 0x0000]
[cpu 0x0000] ******************************* Summary *******************************
[cpu 0x0000] > Test suite 'Template'
[cpu 0x0000]                                                                 Passed
[cpu 0x0000] =================================
[cpu 0x0000] Tests Skipped : 0
[cpu 0x0000] Tests Passed  : 2
[cpu 0x0000] Tests Failed  : 0
[cpu 0x0000] Tests Crashed : 0
[cpu 0x0000] Total tests   : 2
[cpu 0x0000] =================================
[cpu 0x0000] NOTICE:  Exiting tests.

Change-Id: I9d52f2da8905962ab1df73d0691846d88622d3b5
Signed-off-by: Sandrine Bailleux <sandrine.bailleux@arm.com>

4 files changed, 86 insertions, 55 deletions

diff --git a/include/lib/tftf_lib.h b/include/lib/tftf_lib.h
index 97b6f41..40955d3 100644
--- a/include/lib/tftf_lib.h
+++ b/include/lib/tftf_lib.h
@@ -36,8 +36,6 @@ typedef enum {
 	TEST_RESULT_MAX
 } test_result_t;
 
-const char *test_result_to_string(test_result_t result);
-
 #define TEST_RESULT_IS_VALID(result) \
 	((result >= TEST_RESULT_MIN) && (result < TEST_RESULT_MAX))
 
diff --git a/tftf/framework/include/tftf.h b/tftf/framework/include/tftf.h
index 8231e28..43f1e7e 100644
--- a/tftf/framework/include/tftf.h
+++ b/tftf/framework/include/tftf.h
@@ -130,7 +130,10 @@ STATUS tftf_testcase_set_result(const test_case_t *testcase,
 */
 STATUS tftf_testcase_get_result(const test_case_t *testcase, TESTCASE_RESULT *result, char *test_output);
 
-void tftf_report_generate(void);
+void print_testsuite_start(const test_suite_t *testsuite);
+void print_test_start(const test_case_t *test);
+void print_test_end(const test_case_t *test);
+void print_tests_summary(void);
 
 /*
  * Exit the TFTF.
diff --git a/tftf/framework/main.c b/tftf/framework/main.c
index 3f94dc9..ea56f2e 100644
--- a/tftf/framework/main.c
+++ b/tftf/framework/main.c
@@ -148,8 +148,14 @@ static void prepare_next_test(void)
 	for (unsigned int i = 0; i < PLATFORM_CORE_COUNT; ++i)
 		test_results[i] = TEST_RESULT_NA;
 
-	NOTICE("Starting unittest '%s - %s'\n",
-		current_testsuite()->name, current_testcase()->name);
+	/* If we're starting a new testsuite, announce it. */
+	test_ref_t test_to_run;
+	tftf_get_test_to_run(&test_to_run);
+	if (test_to_run.testcase_idx == 0) {
+		print_testsuite_start(current_testsuite());
+	}
+
+	print_test_start(current_testcase());
 
 	/* Program the watchdog */
 	tftf_platform_watchdog_set();
@@ -263,21 +269,18 @@ static unsigned int close_test(void)
 	assert(tftf_get_ref_cnt() == 0);
 
 	/* Save test result in NVM */
-	test_result_t overall_test_result = get_overall_test_result();
 	tftf_testcase_set_result(current_testcase(),
-				overall_test_result,
+				get_overall_test_result(),
 				0);
 
-	NOTICE("Unittest '%s - %s' complete. Result: %s\n",
-	       current_testsuite()->name, current_testcase()->name,
-	       test_result_to_string(overall_test_result));
+	print_test_end(current_testcase());
 
 	/* The test is finished, let's move to the next one (if any) */
 	next_test = advance_to_next_test();
 
 	/* If this was the last test then report all results */
 	if (!next_test) {
-		tftf_report_generate();
+		print_tests_summary();
 		tftf_clean_nvm();
 		return 1;
 	} else {
@@ -551,7 +554,7 @@ void __dead2 tftf_cold_boot_main(void)
 		NOTICE("Resuming interrupted test session\n");
 		rc = resume_test_session();
 		if (rc < 0) {
-			tftf_report_generate();
+			print_tests_summary();
 			tftf_clean_nvm();
 			tftf_exit();
 		}
diff --git a/tftf/framework/report.c b/tftf/framework/report.c
index faae34b..663caf3 100644
--- a/tftf/framework/report.c
+++ b/tftf/framework/report.c
@@ -6,70 +6,97 @@
 
 #include <assert.h>
 #include <debug.h>
-#include <platform_def.h>  /* For TESTCASE_OUTPUT_MAX_SIZE */
 #include <stdio.h>
-#include <string.h>
+#include <stdbool.h>
 #include <tftf.h>
 
-static unsigned int total_tests;
-static unsigned int tests_stats[TEST_RESULT_MAX];
-
-static void tftf_update_tests_statistics(test_result_t result)
-{
-	assert(TEST_RESULT_IS_VALID(result));
-	total_tests++;
-	tests_stats[result]++;
-}
-
 static const char *test_result_strings[TEST_RESULT_MAX] = {
 	"Skipped", "Passed", "Failed", "Crashed",
 };
 
-const char *test_result_to_string(test_result_t result)
+static const char *test_result_to_string(test_result_t result)
 {
 	assert(TEST_RESULT_IS_VALID(result));
 	return test_result_strings[result];
 }
 
-void tftf_report_generate(void)
+void print_testsuite_start(const test_suite_t *testsuite)
 {
-	unsigned i, j;
-	const test_case_t *testcases;
-	TESTCASE_RESULT testcase_result;
-	char test_output[TESTCASE_OUTPUT_MAX_SIZE];
-	STATUS status;
-
-	/* Extract the result of all the testcases */
-	printf("========== TEST REPORT ==========\n");
-	for (i = 0; testsuites[i].name != NULL; i++) {
-		printf("# Test suite '%s':\n", testsuites[i].name);
-		testcases = testsuites[i].testcases;
-
-		for (j = 0; testcases[j].name != NULL; j++) {
-			status = tftf_testcase_get_result(&testcases[j], &testcase_result, test_output);
-			if (status != STATUS_SUCCESS) {
-				printf("Failed to get test result.\n");
+	mp_printf("--\n");
+	mp_printf("Running test suite '%s'\n", testsuite->name);
+	mp_printf("Description: %s\n", testsuite->description);
+	mp_printf("\n");
+}
+
+void print_test_start(const test_case_t *test)
+{
+	mp_printf("> Executing '%s'\n", test->name);
+}
+
+void print_test_end(const test_case_t *test)
+{
+	TESTCASE_RESULT result;
+	char output[TESTCASE_OUTPUT_MAX_SIZE];
+
+	tftf_testcase_get_result(test, &result, output);
+
+	mp_printf("  TEST COMPLETE %54s\n",
+		  test_result_to_string(result.result));
+	if (strlen(output) != 0) {
+		mp_printf("%s", output);
+	}
+	mp_printf("\n");
+}
+
+void print_tests_summary(void)
+{
+	int total_tests = 0;
+	int tests_stats[TEST_RESULT_MAX] = { 0 };
+
+	mp_printf("******************************* Summary *******************************\n");
+
+	/* Go through the list of test suites. */
+	for (int i = 0; testsuites[i].name != NULL; i++) {
+		bool passed = true;
+
+		mp_printf("> Test suite '%s'\n", testsuites[i].name);
+
+		const test_case_t *testcases = testsuites[i].testcases;
+
+		/* Go through the list of tests inside this test suite. */
+		for (int j = 0; testcases[j].name != NULL; j++) {
+			TESTCASE_RESULT result;
+			char output[TESTCASE_OUTPUT_MAX_SIZE];
+
+			if (tftf_testcase_get_result(&testcases[j], &result,
+					output) != STATUS_SUCCESS) {
+				mp_printf("Failed to get test result.\n");
 				continue;
 			}
 
-			tftf_update_tests_statistics(testcase_result.result);
-			/* TODO: print test duration */
-			printf("\t - %s: %s\n", testcases[j].name,
-				test_result_to_string(testcase_result.result));
+			assert(TEST_RESULT_IS_VALID(result.result));
 
-			if (strlen(test_output) != 0) {
-				printf("--- output ---\n");
-				printf("%s", test_output);
-				printf("--------------\n");
+			/*
+			 * Consider that a test suite passed if all of its
+			 * tests passed or were skipped.
+			 */
+			if ((result.result != TEST_RESULT_SUCCESS) &&
+			    (result.result != TEST_RESULT_SKIPPED)) {
+				passed = false;
 			}
+
+			total_tests++;
+			tests_stats[result.result]++;
 		}
+		mp_printf("%70s\n", passed ? "Passed" : "Failed");
 	}
-	printf("=================================\n");
 
-	for (i = TEST_RESULT_MIN; i < TEST_RESULT_MAX; i++) {
-		printf("Tests %-8s: %d\n",
+	mp_printf("=================================\n");
+
+	for (int i = TEST_RESULT_MIN; i < TEST_RESULT_MAX; i++) {
+		mp_printf("Tests %-8s: %d\n",
 			test_result_to_string(i), tests_stats[i]);
 	}
-	printf("%-14s: %d\n", "Total tests", total_tests);
-	printf("=================================\n");
+	mp_printf("%-14s: %d\n", "Total tests", total_tests);
+	mp_printf("=================================\n");
 }