tests: Add gcov support

If the -C flag is given to sanitycheck, generate gcov files for unit
tests and render them with lcov.

Signed-off-by: Jaakko Hannikainen <jaakko.hannikainen@intel.com>
Change-Id: Ic25eae6a3cfc2c45595bd6aa235df2c483aaf6ec

1 files changed, 40 insertions, 9 deletions

diff --git a/scripts/sanitycheck b/scripts/sanitycheck
index ea4f24064..3ae2f3385 100755
--- a/scripts/sanitycheck
+++ b/scripts/sanitycheck
@@ -273,7 +273,7 @@ class Handler:
         return ret
 
 class UnitHandler(Handler):
-    def __init__(self, name, outdir, run_log, valgrind_log, timeout):
+    def __init__(self, name, sourcedir, outdir, run_log, valgrind_log, timeout):
         """Constructor
 
         @param name Arbitrary name of the created thread
@@ -286,6 +286,7 @@ class UnitHandler(Handler):
         super().__init__(name, outdir, run_log, timeout, True)
 
         self.timeout = timeout
+        self.sourcedir = sourcedir
         self.outdir = outdir
         self.run_log = run_log
         self.valgrind_log = valgrind_log
@@ -316,6 +317,8 @@ class UnitHandler(Handler):
                 out_state = "timeout"
                 self.returncode = 1
 
+        returncode = subprocess.call(["GCOV_PREFIX=" + self.outdir, "gcov", self.sourcedir, "-s", self.outdir], shell=True)
+
         self.set_state(out_state, {})
 
 class QEMUHandler(Handler):
@@ -766,24 +769,26 @@ class MakeGenerator:
         self.goals[name] = MakeGoal(name, text, q, self.logfile, build_logfile,
                                     run_logfile, qemu_logfile)
 
-    def add_unit_goal(self, name, directory, outdir, args, timeout=30):
+    def add_unit_goal(self, name, directory, outdir, args, timeout=30, coverage=False):
         self._add_goal(outdir)
         build_logfile = os.path.join(outdir, "build.log")
         run_logfile = os.path.join(outdir, "run.log")
         qemu_logfile = os.path.join(outdir, "qemu.log")
         valgrind_logfile = os.path.join(outdir, "valgrind.log")
+        if coverage:
+                args += ["COVERAGE=1"]
 
         # we handle running in the UnitHandler class
         text = (self._get_rule_header(name) +
                 self._get_sub_make(name, "building", directory,
                                    outdir, build_logfile, args) +
                 self._get_rule_footer(name))
-        q = UnitHandler(name, outdir, run_logfile, valgrind_logfile, timeout)
+        q = UnitHandler(name, directory, outdir, run_logfile, valgrind_logfile, timeout)
         self.goals[name] = MakeGoal(name, text, q, self.logfile, build_logfile,
                                     run_logfile, valgrind_logfile)
 
 
-    def add_test_instance(self, ti, build_only=False, enable_slow=False,
+    def add_test_instance(self, ti, build_only=False, enable_slow=False, coverage=False,
                           extra_args=[]):
         """Add a goal to build/test a TestInstance object
 
@@ -800,7 +805,7 @@ class MakeGenerator:
                                args, ti.test.timeout)
         elif ti.test.type == "unit":
             self.add_unit_goal(ti.name, ti.test.code_location, ti.outdir,
-                               args, ti.test.timeout)
+                               args, ti.test.timeout, coverage)
         else:
             self.add_build_goal(ti.name, ti.test.code_location, ti.outdir, args)
 
@@ -1195,7 +1200,7 @@ class TestInstance:
         out directory used is <outdir>/<platform>/<test case name>
     """
     def __init__(self, test, platform, base_outdir, build_only=False,
-                 slow=False):
+                 slow=False, coverage=False):
         self.test = test
         self.platform = platform
         self.name = os.path.join(platform.name, test.path)
@@ -1237,7 +1242,7 @@ def defconfig_cb(context, goals, goal):
 class TestSuite:
     config_re = re.compile('(CONFIG_[A-Z0-9_]+)[=]\"?([^\"]*)\"?$')
 
-    def __init__(self, arch_root, testcase_roots, outdir):
+    def __init__(self, arch_root, testcase_roots, outdir, coverage):
         # Keep track of which test cases we've filtered out and why
         discards = {}
         self.arches = {}
@@ -1247,6 +1252,7 @@ class TestSuite:
         self.instances = {}
         self.goals = None
         self.discards = None
+        self.coverage = coverage
 
         arch_root = os.path.abspath(arch_root)
 
@@ -1521,7 +1527,7 @@ class TestSuite:
 
         mg = MakeGenerator(self.outdir, asserts=enable_asserts)
         for i in self.instances.values():
-            mg.add_test_instance(i, build_only, enable_slow, extra_args)
+            mg.add_test_instance(i, build_only, enable_slow, self.coverage, extra_args)
         self.goals = mg.execute(cb, cb_context)
 
         # Parallelize size calculation
@@ -1735,6 +1741,8 @@ def parse_arguments():
             help="Extra arguments to pass to the build when compiling test "
                  "cases. May be called multiple times. These will be passed "
                  "in after any sanitycheck-supplied options.")
+    parser.add_argument("-C", "--coverage", action="store_true",
+            help="Scan for unit test coverage with gcov + lcov.")
 
     return parser.parse_args()
 
@@ -1808,6 +1816,25 @@ def size_report(sc):
              (sc.rom_size, sc.ram_size))
     info("")
 
+def generate_coverage(outdir, ignores):
+    with open(os.path.join(outdir, "coverage.log"), "a") as coveragelog:
+        coveragefile = os.path.join(outdir, "coverage.info")
+        ztestfile = os.path.join(outdir, "ztest.info")
+        subprocess.call(["lcov", "--capture", "--directory", outdir,
+                         "--output-file", coveragefile], stdout=coveragelog)
+        # We want to remove tests/* and tests/ztest/test/* but save tests/ztest
+        subprocess.call(["lcov", "--extract", coveragefile,
+                        os.path.join(ZEPHYR_BASE, "tests", "ztest", "*"),
+                        "--output-file", ztestfile], stdout=coveragelog)
+        subprocess.call(["lcov", "--remove", ztestfile,
+                        os.path.join(ZEPHYR_BASE, "tests/ztest/test/*"),
+                        "--output-file", ztestfile], stdout=coveragelog)
+        for i in ignores:
+            subprocess.call(["lcov", "--remove", coveragefile, i,
+                            "--output-file", coveragefile], stdout=coveragelog)
+        subprocess.call(["genhtml", "-output-directory",
+                        os.path.join(outdir, "coverage"),
+                        coveragefile, ztestfile], stdout=coveragelog)
 
 def main():
     start_time = time.time()
@@ -1833,7 +1860,7 @@ def main():
         args.testcase_root = [os.path.join(ZEPHYR_BASE, "tests"),
                               os.path.join(ZEPHYR_BASE, "samples")]
 
-    ts = TestSuite(args.arch_root, args.testcase_root, args.outdir)
+    ts = TestSuite(args.arch_root, args.testcase_root, args.outdir, args.coverage)
     discards = ts.apply_filters(args.platform, args.arch, args.tag, args.config,
                                 args.test, args.only_failed, args.all,
                                 args.platform_limit, toolchain, args.extra_args)
@@ -1908,6 +1935,10 @@ def main():
                   str(goal.metrics["mismatched"])))
             failed += 1
 
+    if args.coverage:
+        info("Generating coverage files...")
+        generate_coverage(args.outdir, ["tests/*", "samples/*"])
+
     info("%s%d of %d%s tests passed with %s%d%s warnings in %d seconds" %
           (COLOR_RED if failed else COLOR_GREEN, len(goals) - failed,
            len(goals), COLOR_NORMAL, COLOR_YELLOW if warnings else COLOR_NORMAL,