Parse and save results in json and csv format

Signed-off-by: Chase Qi <chase.qi@linaro.org>

3 files changed, 143 insertions, 109 deletions

diff --git a/lava-local-test.py b/lava-local-test.py
index 99203ca..6551956 100644
--- a/lava-local-test.py
+++ b/lava-local-test.py
@@ -1,17 +1,22 @@
 import sys
 import os
+import glob
 import shutil
 import time
 import re
 import yaml
+import json
+import csv
 import subprocess
 import pexpect
 from uuid import uuid4
 
 
 HOME = os.environ['HOME']
-LAVA_PATH = HOME + '/lava_test_output'
-test_definition = 'smoke.yaml'
+LAVA_PATH = HOME + '/lava_output'
+git_repo = 'https://git.linaro.org/qa/test-definitions.git'
+test_def = 'ubuntu/openssl.yaml'
+# test_def = 'ubuntu/smoke-tests-basic.yaml'
 
 
 class TestDefinition(object):
@@ -19,12 +24,12 @@ class TestDefinition(object):
     Analysis and convert test definition.
     """
 
-    def __init__(self, test_definition, test_path):
-        self.test_definition = test_definition
+    def __init__(self):
+        self.test_def = test_path + '/' + test_def
         self.test_path = test_path
         # Read the YAML to create a testdef dict
-        with open(self.test_definition, 'r') as test_file:
-            self.testdef = yaml.safe_load(test_file)
+        with open(self.test_def, 'r') as f:
+            self.testdef = yaml.safe_load(f)
 
     def definition(self):
         with open('%s/testdef.yaml' % self.test_path, 'w') as f:
@@ -38,33 +43,43 @@ class TestDefinition(object):
         pass
 
     def run(self):
-        with open('%s/run.sh' % self.test_path, 'a') as runsh:
-            runsh.write('set -e\n')
-            runsh.write('export TESTRUN_ID=%s\n' % self.testdef['metadata']['name'])
-            runsh.write('cd %s\n' % self.test_path)
-            runsh.write('UUID=`cat uuid`\n')
-            runsh.write('echo "<LAVA_SIGNAL_STARTRUN $TESTRUN_ID $UUID>"\n')
+        with open('%s/run.sh' % self.test_path, 'a') as f:
+            f.write('set -e\n')
+            f.write('export TESTRUN_ID=%s\n' % self.testdef['metadata']['name'])
+            f.write('cd %s\n' % self.test_path)
+            f.write('UUID=`cat uuid`\n')
+            f.write('echo "<LAVA_SIGNAL_STARTRUN $TESTRUN_ID $UUID>"\n')
             steps = self.testdef['run'].get('steps', [])
             if steps:
                 for cmd in steps:
                     if '--cmd' in cmd or '--shell' in cmd:
                         cmd = re.sub(r'\$(\d+)\b', r'\\$\1', cmd)
-                    runsh.write('%s\n' % cmd)
-            runsh.write('echo "<LAVA_SIGNAL_ENDRUN $TESTRUN_ID $UUID>"\n')
+                    f.write('%s\n' % cmd)
+            f.write('echo "<LAVA_SIGNAL_ENDRUN $TESTRUN_ID $UUID>"\n')
 
     def parameters(self):
         pass
 
+    def return_pattern(self):
+        if 'parse' in self.testdef:
+            return self.testdef['parse']['pattern']
+        else:
+            return None
+
 
 class TestSetup(object):
-    def __init__(self, lava_path, test_definition):
-        self.lava_path = lava_path
-        self.test_name = os.path.splitext(test_definition)[0]
-        self.uuid = str(uuid4())
-        self.test_uuid = self.test_name + '_' + self.uuid
-        self.bin_path = lava_path + '/bin'
-        self.test_path = lava_path + '/tests/' + self.test_uuid
-        self.result_path = lava_path + '/results/' + self.test_uuid
+    def __init__(self):
+        self.repo_name = repo_name
+        self.test_name = test_name
+        self.uuid = uuid
+        self.test_uuid = test_uuid
+        self.lava_path = LAVA_PATH
+        self.bin_path = bin_path
+        self.test_path = test_path
+
+    def copy_test_repo(self):
+        shutil.rmtree(self.test_path, ignore_errors=True)
+        shutil.copytree(self.repo_name, self.test_path, symlinks=True)
 
     def create_dir(self):
         if not os.path.exists(self.test_path):
@@ -75,6 +90,7 @@ class TestSetup(object):
             f.write(self.test_path)
 
     def copy_bin_files(self):
+        # TODO: handle distro specific files.
         shutil.rmtree(self.bin_path, ignore_errors=True)
         shutil.copytree('lava_test_shell', self.bin_path, symlinks=True)
 
@@ -82,16 +98,12 @@ class TestSetup(object):
         with open('%s/uuid' % self.test_path, 'w') as f:
             f.write(self.uuid)
 
-    def get_test_path(self):
-        return self.test_path
-
-    def get_result_path(self):
-        return self.result_path
-
 
 class TestRunner(object):
-    def __init__(self, lava_path):
-        self.lava_path = lava_path
+    def __init__(self):
+        self.lava_path = LAVA_PATH
+        self.test_uuid = test_uuid
+        print('\n About to run %s' % self.test_uuid)
         self.child = pexpect.spawn('%s/bin/lava-test-runner %s' % (self.lava_path, self.lava_path))
 
     def check_output(self):
@@ -101,22 +113,115 @@ class TestRunner(object):
                 print(self.child.before)
             except pexpect.EOF:
                 break
+        print('%s test finished.\n' % self.test_uuid)
+
+
+class ResultPaser(object):
+    def __init__(self):
+        self.result_path = result_path
+        # Fix result path with timestamp added by lava-test-runner.
+        self.result_path = glob.glob('%s-[0-9]*' % self.result_path)[0]
+        # TODO: handle result parser dfined in testdef
+        self.pattern = pattern
+        self.metrics = []
+        self.results = {}
+        self.test_uuid = self.result_path.split('/')[-1]
+        self.results['test'] = self.test_uuid.split('_')[0]
+        self.results['id'] = self.test_uuid.split('_')[1]
+
+    def run(self):
+        if not self.pattern:
+            self.parse_lava_test_case()
+        else:
+            self.parse_pattern()
+
+        self.dict_to_json()
+        self.dict_to_csv()
+        print('Result files saved to: %s\n' % self.result_path)
+
+    def parse_lava_test_case(self):
+        with open('%s/stdout.log' % self.result_path, 'r') as f:
+            for line in f:
+                if re.match(r'\<LAVA_SIGNAL_TESTCASE TEST_CASE_ID=.*', line):
+                    line = line.strip('\n').strip('<>').split(' ')
+                    self.data = {'test_case_id': '',
+                                 'result': '',
+                                 'measurement': '',
+                                 'units': ''}
+
+                    for string in line:
+                        parts = string.split('=')
+                        if len(parts) == 2:
+                            key, value = parts
+                            key = key.lower()
+                            self.data[key] = value
+
+                    self.metrics.append(self.data.copy())
+
+        self.results['metrics'] = self.metrics
+
+    def parse_pattern(self):
+        print self.pattern
+        with open('%s/stdout.log' % self.result_path, 'r') as f:
+            for line in f:
+                self.data = {}
+                m = re.match(r'%s' % self.pattern, line)
+                if m:
+                    self.data = m.groupdict()
+
+                    for x in ['measurement', 'units']:
+                        if x not in self.data:
+                            self.data[x] = ''
+
+                    self.metrics.append(self.data.copy())
+
+        self.results['metrics'] = self.metrics
+
+    def dict_to_json(self):
+        with open('%s/results.json' % self.result_path, 'w') as f:
+            json.dump(self.results, f, indent=4)
+
+    def dict_to_csv(self):
+        with open('%s/results.csv' % self.result_path, 'w') as f:
+            fieldnames = ['test_case_id', 'result', 'measurement', 'units']
+            writer = csv.DictWriter(f, fieldnames=fieldnames)
+
+            writer.writeheader()
+            for metric in self.results['metrics']:
+                writer.writerow(metric)
+
+
+# Pre-config.
+uuid = str(uuid4())
+test_name = os.path.splitext(test_def.split('/')[-1])[0]
+test_uuid = test_name + '_' + uuid
+bin_path = LAVA_PATH + '/bin'
+test_path = LAVA_PATH + '/tests/' + test_uuid
+result_path = LAVA_PATH + '/results/' + test_uuid
+
+repo_name = os.path.splitext(git_repo.split('/')[-1])[0]
+if not os.path.exists(repo_name):
+    subprocess.call(['git', 'clone', git_repo])
 
 # Create a hierarchy of directories and files needed.
-test_setup = TestSetup(LAVA_PATH, test_definition)
-test_setup.create_dir()
-test_setup.create_test_runner_conf()
-test_setup.copy_bin_files()
-test_setup.create_uuid_file()
-test_path = test_setup.get_test_path()
+setup = TestSetup()
+setup.copy_test_repo()
+setup.create_test_runner_conf()
+setup.copy_bin_files()
+setup.create_uuid_file()
 
 # Convert test definition to the files needed by lava-test-runner.
-test_def = TestDefinition(test_definition, test_path)
+test_def = TestDefinition()
 test_def.definition()
 test_def.metadata()
 test_def.install()
 test_def.run()
+pattern = test_def.return_pattern()
 
 # Test run.
-test_run = TestRunner(LAVA_PATH)
+test_run = TestRunner()
 test_run.check_output()
+
+# Parse test output, save to csv file.
+parser = ResultPaser()
+parser.run()
diff --git a/pi.yaml b/pi.yaml
deleted file mode 100644
index 265dddf..0000000
--- a/pi.yaml
+++ /dev/null
@@ -1,44 +0,0 @@
-metadata:
-    format: Lava-Test Test Definition 1.0
-    name: pi-stress-test
-    description: "PI-Stress-test for POSIX Priority Inheritance mutexes. It's a stress test that
-                  checks Priority Inheritence Mutexes and their ability to avoid Priority Inversion
-                  from occuring by running groups of threads that cause Priority Inversions."
-    maintainer:
-        - chase.qi@linaro.org
-    os:
-        - ubuntu
-    scope:
-        - functional
-        - preempt-rt
-    devices:
-        - panda
-        - panda-es
-        - arndale
-        - vexpress-a9
-        - vexpress-tc2
-        - beaglebone-black
-        - d01
-        - rtsm_fvp_base-aemv8a
-        - juno
-    environment:
-        - lava-test-shell
-
-install:
-    deps:
-        - rt-tests
-params:
-    # Length of the test run in seconds
-    DURATION: 300
-    # The number of inversion groups to run. By default pi_stress will detect the number of
-    # processors and determine the number of inversion groups automatically.
-    GROUP: 'default'
-    # Set MLOCKALL to ture to lock current and future memory
-    MLOCKALL: 'true'
-    # Set RR to true if you need use SCHED_RR for test threads. The
-    # default is to run the inversion threads as SCHED_FIFO
-    RR: 'false'
-
-run:
-    steps:
-        - ./ubuntu/scripts/pi-stress-test.py $DURATION $GROUP $MLOCKALL $RR
diff --git a/smoke.yaml b/smoke.yaml
deleted file mode 100644
index 6a33377..0000000
--- a/smoke.yaml
+++ /dev/null
@@ -1,27 +0,0 @@
-metadata:
-    format: Lava-Test Test Definition 1.0
-    name: smoke-tests-basic
-    description: "Basic system test command for Linaro Ubuntu images. The test runs basic commands
-                  like pwd, uname, vmstat, ifconfig, lscpu, lsusb and lsb_release."
-    maintainer:
-        - milosz.wasilewski@linaro.org
-    os:
-        - ubuntu
-    scope:
-        - functional
-    devices:
-        - panda
-        - panda-es
-        - arndale
-        - vexpress-a9
-        - vexpress-tc2
-        - juno
-
-run:
-    steps:
-        - lava-test-case linux-linaro-ubuntu-pwd --shell pwd
-        - lava-test-case linux-linaro-ubuntu-uname --shell uname -a
-        - lava-test-case linux-linaro-ubuntu-vmstat --shell vmstat
-        - lava-test-case linux-linaro-ubuntu-ifconfig --shell ifconfig -a
-        - lava-test-case linux-linaro-ubuntu-lscpu --shell lscpu
-        - lava-test-case linux-linaro-ubuntu-lsb_release --shell lsb_release -a