Create metadata on the number of test definitions

Display the metadata on the results page for the job.
Allow for use to compare with the number of test definitions
for which there are results. Handle multinode jobs to
prevent duplicate TestData in future. Closes: #2259

Change-Id: I3a404e2e30eaa28fa1f63930f99794322a2b171f

5 files changed, 194 insertions, 45 deletions

diff --git a/lava_results_app/dbutils.py b/lava_results_app/dbutils.py
index 374197048..d947a3315 100644
--- a/lava_results_app/dbutils.py
+++ b/lava_results_app/dbutils.py
@@ -28,21 +28,11 @@ from lava_results_app.models import (
     ActionData,
     MetaType,
 )
+from django.core.exceptions import MultipleObjectsReturned
 from lava_dispatcher.pipeline.action import Timeout
 # pylint: disable=no-member
 
 
-METADATA_MAPPING_DESCRIPTION = {
-    "boot.commands": ["job", "actions", "boot", "commands"],
-    "boot.method": ["job", "actions", "boot", "method"],
-    "boot.type": ["job", "actions", "boot", "type"],
-    "deploy.os": ["job", "actions", "deploy", "os"],
-    "deploy.ramdisk-type": ["job", "actions", "deploy", "ramdisk-type"],
-    "target.hostname": ["device", "hostname"],
-    "target.device_type": ["device", "device_type"]
-}
-
-
 def _test_case(name, suite, result, testset=None, testshell=False):
     """
     Create a TestCase for the specified name and result
@@ -125,11 +115,11 @@ def _check_for_testset(result_dict, suite):
     return testset
 
 
-def map_scanned_results(scanned_dict, job):
+def map_scanned_results(scanned_dict, job):  # pylint: disable=too-many-branches
     """
     Sanity checker on the logged results dictionary
     :param scanned_dict: results logged via the slave
-    :param suite: the current test suite
+    :param job: the current test job
     :return: False on error, else True
     """
     logger = logging.getLogger('dispatcher-master')
@@ -172,21 +162,70 @@ def map_scanned_results(scanned_dict, job):
     return True
 
 
-def _get_nested_value(data, mapping):
-    # get the value from a nested dictionary based on keys given in 'mapping'.
-    value = data
-    for key in mapping:
-        try:
-            value = value[key]
-        except TypeError:
-            # check case when nested value is list and not dict.
-            for item in value:
-                if key in item:
-                    value = item[key]
-        except KeyError:
-            return None
+def _get_job_metadata(data):  # pylint: disable=too-many-branches
+    if not isinstance(data, list):
+        return None
+    retval = {}
+    for action in data:
+        deploy = [reduce(dict.get, ['deploy'], action)]
+        count = 0
+        for block in deploy:
+            if not block:
+                continue
+            namespace = block.get('namespace', None)
+            prefix = "deploy.%d.%s" % (count, namespace) if namespace else 'deploy.%d' % count
+            value = block.get('method', None)
+            if value:
+                retval['%s.method' % prefix] = value
+                count += 1
+        boot = [reduce(dict.get, ['boot'], action)]
+        count = 0
+        for block in boot:
+            if not block:
+                continue
+            namespace = block.get('namespace', None)
+            prefix = "boot.%d.%s" % (count, namespace) if namespace else 'boot.%d' % count
+            value = block.get('commands', None)
+            if value:
+                retval['%s.commands' % prefix] = value
+            value = block.get('method', None)
+            if value:
+                retval['%s.method' % prefix] = value
+            value = block.get('type', None)
+            if value:
+                retval['%s.type' % prefix] = value
+            count += 1
+        test = [reduce(dict.get, ['test'], action)]
+        count = 0
+        for block in test:
+            if not block:
+                continue
+            namespace = block.get('namespace', None)
+            definitions = [reduce(dict.get, ['definitions'], block)][0]
+            for definition in definitions:
+                if definition['from'] == 'inline':
+                    # an inline repo without test cases will not get reported.
+                    if 'lava-test-case' in [reduce(dict.get, ['repository', 'run', 'steps'], definition)][0]:
+                        prefix = "test.%d.%s" % (count, namespace) if namespace else 'test.%d' % count
+                        retval['%s.inline' % prefix] = definition['name']
+                else:
+                    prefix = "test.%d.%s" % (count, namespace) if namespace else 'test.%d' % count
+                    # FIXME: what happens with remote definition without lava-test-case?
+                    retval['%s.definition.name' % prefix] = definition['name']
+                    retval['%s.definition.path' % prefix] = definition['path']
+                    retval['%s.definition.from' % prefix] = definition['from']
+                    retval['%s.definition.repository' % prefix] = definition['repository']
+                count += 1
+    return retval
+
 
-    return value
+def _get_device_metadata(data):
+    hostname = data.get('hostname', None)
+    devicetype = data.get('device_type', None)
+    return {
+        'target.hostname': hostname,
+        'target.device_type': devicetype
+    }
 
 
 def build_action(action_data, testdata, submission):
@@ -215,6 +254,7 @@ def build_action(action_data, testdata, submission):
     if 'max_retries' in action_data:
         max_retry = action_data['max_retries']
 
+    # maps the static testdata derived from the definition to the runtime pipeline construction
     action = ActionData.objects.create(
         action_name=action_data['name'],
         action_level=action_data['level'],
@@ -254,16 +294,26 @@ def map_metadata(description, job):
     except yaml.YAMLError as exc:
         logger.exception("[%s] %s", job.id, exc)
         return False
-    testdata = TestData.objects.create(testjob=job)
+    try:
+        testdata, created = TestData.objects.get_or_create(testjob=job)
+    except MultipleObjectsReturned:
+        # only happens for small number of jobs affected by original bug.
+        logger.info("[%s] skipping alteration of duplicated TestData", job.id)
+        return False
+    if not created:
+        # prevent updates of existing TestData
+        return False
     testdata.save()
-    # Add metadata from description data.
-    for key in METADATA_MAPPING_DESCRIPTION:
-        value = _get_nested_value(
-            description_data,
-            METADATA_MAPPING_DESCRIPTION[key]
-        )
-        if value:
-            testdata.attributes.create(name=key, value=value)
+
+    # get job-action metadata
+    action_values = _get_job_metadata(description_data['job']['actions'])
+    for key, value in action_values.items():
+        testdata.attributes.create(name=key, value=value)
+
+    # get metadata from device
+    device_values = _get_device_metadata(description_data['device'])
+    for key, value in device_values.items():
+        testdata.attributes.create(name=key, value=value)
 
     # Add metadata from job submission data.
     if "metadata" in submission_data:
@@ -292,7 +342,7 @@ def export_testcase(testcase):
     Returns string versions of selected elements of a TestCase
     Unicode causes issues with CSV and can complicate YAML parsing
     with non-python parsers.
-    :param testcases: list of TestCase objects
+    :param testcase: list of TestCase objects
     :return: Dictionary containing relevant information formatted for export
     """
     actiondata = testcase.action_data
diff --git a/lava_results_app/templates/lava_results_app/job.html b/lava_results_app/templates/lava_results_app/job.html
index 3e7e89e64..3b0c929c8 100644
--- a/lava_results_app/templates/lava_results_app/job.html
+++ b/lava_results_app/templates/lava_results_app/job.html
@@ -1,5 +1,6 @@
 {% extends "layouts/content-bootstrap.html" %}
 {% load i18n %}
+{% load utils %}
 {% load django_tables2 %}
 
 {% block content %}
@@ -27,15 +28,50 @@
  <div class="panel-group" id="results_accordion">
   <div class="panel panel-default">
     <div class="panel-heading">
-      <h4 class="panel-title"><a data-toggle="collapse" data-parent="#results_accordion" href="#metadata_collapse">
+      <h4><a data-toggle="collapse" data-parent="#results_accordion" href="#metadata_collapse">
         Metadata
-      </a></h4>
+      </a>
+      </h4>
     </div>
+  {% spaceless %}
     <div id="metadata_collapse" class="panel-collapse collapse">
       <div class="panel-body">
-          <pre>{{ metadata }}</pre>
+      <p>The <b>key</b> is the name of the metadata named attribute which can be used in query conditions.
+          Values relating to devices, device-types and URLs are formatted as links. Due to
+          the variation between different git server interfaces, it is not possible to construct
+          a full URL to the test definition file. The commit id of the test definition is part of
+          the <i>lava</i> results for the job.
+      </p>
+      <p>Attributes relating to items which can repeat within a single job include a number representing
+          the sequence of that item within the job. For example, <i>boot.0.method</i> is the name of an
+          attribute containing information on the first boot method used in the job. <i>boot.0.commands</i>
+          would be an attribute containing information on the commands used by the first boot method in
+          the job.
+      </p>
+      <p>Metadata submitted as part of the job submission is also included, if present.</p>
+        <dl class="dl-horizontal">
+          {% for key, value in metadata.items %}
+              {% if 'target' in key %}
+                  <dt>{{ key|metadata_key}}</dt>
+                  <dd>{{ key|markup_metadata:value }}</dd>
+              {% elif 'deploy' in key %}
+                  <dt>{{ key|metadata_key}}</dt>
+                  <dd>{{ value }}</dd>
+              {% elif 'boot' in key %}
+                  <dt>{{ key|metadata_key}}</dt>
+                  <dd>{{ value }}</dd>
+              {% elif 'test' in key %}
+                  <dt>{{ key|metadata_key}}</dt>
+                  <dd>{{ key|markup_metadata:value }}</dd>
+              {% else %}
+                  <dt>{{ key }}</dt>
+                  <dd>{{ value }}</dd>
+              {% endif %}
+          {% endfor %}
+        </dl>
       </div>
     </div>
+  {% endspaceless %}
   </div>
  </div>
 
diff --git a/lava_results_app/tests/test_metadata.py b/lava_results_app/tests/test_metadata.py
index 31795844a..3fb88fdc0 100644
--- a/lava_results_app/tests/test_metadata.py
+++ b/lava_results_app/tests/test_metadata.py
@@ -5,7 +5,12 @@ from lava_scheduler_app.models import (
     TestJob,
     Device,
 )
-from lava_results_app.dbutils import map_metadata, testcase_export_fields, export_testcase
+from lava_results_app.dbutils import (
+    map_metadata,
+    _get_job_metadata, _get_device_metadata,  # pylint: disable=protected-access
+    testcase_export_fields,
+    export_testcase,
+)
 from lava_results_app.models import ActionData, MetaType, TestData, TestCase, TestSuite
 from lava_dispatcher.pipeline.parser import JobParser
 from lava_dispatcher.pipeline.device import PipelineDevice
@@ -85,3 +90,35 @@ class TestMetaTypes(TestCaseWithFactory):
         action_data.timeout = 300
         action_data.save(update_fields=['timeout'])
         self.assertEqual(action_data.timeout, 300)
+
+    def test_repositories(self):
+        job = TestJob.from_yaml_and_user(
+            self.factory.make_job_yaml(), self.user)
+        job_def = yaml.load(job.definition)
+        job_ctx = job_def.get('context', {})
+        device = Device.objects.get(hostname='fakeqemu1')
+        device_config = device.load_device_configuration(job_ctx, system=False)  # raw dict
+        parser = JobParser()
+        obj = PipelineDevice(device_config, device.hostname)
+        pipeline_job = parser.parse(job.definition, obj, job.id, None, None, None, output_dir='/tmp')
+        pipeline_job.pipeline.validate_actions()
+        pipeline = pipeline_job.describe()
+        retval = _get_device_metadata(pipeline['device'])
+        self.assertEqual(
+            retval,
+            {'target.hostname': 'fakeqemu1', 'target.device_type': 'qemu'}
+        )
+        retval = _get_job_metadata(pipeline['job']['actions'])
+        self.assertEqual(
+            retval,
+            {
+                'test.1.definition.from': 'git',
+                'test.0.definition.repository': 'git://git.linaro.org/qa/test-definitions.git',
+                'test.0.definition.name': 'smoke-tests',
+                'test.1.definition.repository': 'http://git.linaro.org/lava-team/lava-functional-tests.git',
+                'boot.0.method': 'qemu',
+                'test.1.definition.name': 'singlenode-advanced',
+                'test.0.definition.from': 'git',
+                'test.0.definition.path': 'ubuntu/smoke-tests-basic.yaml',
+                'test.1.definition.path': 'lava-test-shell/single-node/singlenode03.yaml'}
+        )
diff --git a/lava_results_app/views/__init__.py b/lava_results_app/views/__init__.py
index 9fd857f8d..b1e95a171 100644
--- a/lava_results_app/views/__init__.py
+++ b/lava_results_app/views/__init__.py
@@ -22,6 +22,7 @@ Keep to just the response rendering functions
 """
 import csv
 import yaml
+from collections import OrderedDict
 from django.template import RequestContext
 from django.http.response import HttpResponse, StreamingHttpResponse
 from django.shortcuts import render_to_response
@@ -92,10 +93,12 @@ def testjob(request, job):
     suite_table = ResultsTable(
         data.get_table_data().filter(job=job)
     )
-    testdata = TestData.objects.get(testjob=job)
-    yaml_dict = {}
+    # some duplicates can exist, so get would fail here and [0] is quicker than try except.
+    testdata = TestData.objects.filter(testjob=job)[0]
+    # FIXME get the actiondata as well, use that to map the testdata test defs to what actually got run.
+    yaml_dict = OrderedDict()
     # hide internal python objects
-    for data in testdata.attributes.all():
+    for data in testdata.attributes.all().order_by('name'):
         yaml_dict[str(data.name)] = str(data.value)
     RequestConfig(request, paginate={"per_page": suite_table.length}).configure(suite_table)
     return render_to_response(
@@ -104,7 +107,7 @@ def testjob(request, job):
             'job': job,
             'job_link': pklink(job),
             'suite_table': suite_table,
-            'metadata': yaml.dump(yaml_dict, default_flow_style=False)
+            'metadata': yaml_dict
         }, RequestContext(request))
 
 
diff --git a/lava_scheduler_app/templatetags/utils.py b/lava_scheduler_app/templatetags/utils.py
index feeb0204b..dd61d33a6 100644
--- a/lava_scheduler_app/templatetags/utils.py
+++ b/lava_scheduler_app/templatetags/utils.py
@@ -200,3 +200,26 @@ def result_name(result_dict):
             ))
     else:
         return None
+
+
+@register.filter()
+def metadata_key(key, index=0):
+    return '.'.join(key.split('.')[index:]).replace('definition.', '')
+
+
+@register.filter()
+def markup_metadata(key, value):
+    if 'target.device_type' in key:
+        return mark_safe("<a href='/scheduler/device_type/%s'>%s</a>" % (value, value))
+    elif 'target.hostname' in key:
+        return mark_safe("<a href='/scheduler/device/%s'>%s</a>" % (value, value))
+    elif 'definition.repository' in key:
+        repo = value.replace('git:', 'http:')
+        return mark_safe("<a href='%s'>%s</a>" % (repo, value))
+    else:
+        return value
+
+
+@register.filter()
+def markup_completion(data):
+    return [key for key, _ in data.items() if 'test' in key]