openembedded: Add test{plans,reports} for SA8155p-ADP platform.

Signed-off-by: Bhupesh Sharma <bhupesh.sharma@linaro.org>
Change-Id: I48ea1b4746cf48725a1e43a4e0c4a044e6c7661d

3 files changed, 257 insertions, 0 deletions

diff --git a/openembedded/boards/sa8155p-adp/Testplan.textile b/openembedded/boards/sa8155p-adp/Testplan.textile
new file mode 100644
index 0000000..022bb15
--- /dev/null
+++ b/openembedded/boards/sa8155p-adp/Testplan.textile
@@ -0,0 +1,66 @@
+{% set description = "Linaro OpenEmbedded RPB release for Qualcomm(tm) SA8155p-ADP IoT platform" %}
+h1{color:blue;}. Test Plan for {{description}}
+
+!{width: 20%; padding: 5px; }https://www.linaro.org/assets/images/Linaro-Logo.svg(Linaro Logo)!
+
+*Date:* {{ obj.metadata.now }}
+
+h1. Executive Summary
+
+This document describes the testing Linaro will carry out when making a release of the {{description}}.
+
+h1. Hardware Platforms
+
+Linaro provide releases for the following platforms from Qualcomm:
+
+table{width: 50%; border:1px solid black; border-spacing: 10px; text-align: left; border-collapse:collapse}.
+{background:#ddd}. |_. Board |_. Board Revision |
+| SA8155p-ADP | ADP Air |
+
+h1. Tests definitions
+
+Linaro uses LAVA test definition syntaxt to create and manage automated and manual tests. To learn more about LAVA test definition, you can refer to LAVA Documentation, in particular: "Writing a Lava-Test Test Definition 1.0¶
+":https://docs.lavasoftware.org/lava/writing-tests.html.
+
+Linaro test cases are archived in the upstream Linaro Test Definitions repository here: "https://github.com/Linaro/test-definitions/":https://github.com/Linaro/test-definitions.
+
+h1. Automated Tests
+
+Linaro runs automated testing in the LAVA lab.
+
+This section describes the automated tests used from the Linaro Test Definitions repository.
+
+{% for test in obj.tests.automated %}
+{% if test.missing %} missing {% endif %}
+h2. {{ loop.index }}. {{ test.name }}
+
+{{ test.description }}
+
+_Test definition:_ "{{ test.path }}":{{ test.repository + '/tree/' + test.revision + '/' + test.path }}
+_Test scope:_ {{ test.scope[0] if test.scope[0] is defined else "n/a" }}
+    {% if test.parameters is defined %}
+table{width: 50%; border:1px solid black; border-spacing: 10px; text-align: left; border-collapse:collapse}.
+{background:#ddd}. |_. Parameter |_. Value |
+        {%- for p in test.parameters %}
+| {{ p }} | {{ test.parameters[p] }} |
+        {%- endfor %}
+    {% endif %}
+
+{% endfor %}
+
+h1. Manual Tests
+
+In addition Linaro runs additional manual tests, when they cannot be automated using the Linaro LAVA lab.
+
+This section describes the manual tests used from the Linaro Test Definitions repository.
+
+{% for test in obj.tests.manual %}
+{% if test.missing %} missing {% endif %}
+h2. {{ loop.index }}. {{ test.name }}
+
+{{ test.description }}
+
+_Test definition:_ "{{ test.path }}":{{ test.repository + '/tree/' + test.revision + '/' + test.path }}
+_Test scope:_ {{ test.scope[0] if test.scope[0] is defined else "n/a" }}
+{% endfor %}
+<hr>
diff --git a/openembedded/boards/sa8155p-adp/Testplan.yaml b/openembedded/boards/sa8155p-adp/Testplan.yaml
new file mode 100644
index 0000000..5a80d4f
--- /dev/null
+++ b/openembedded/boards/sa8155p-adp/Testplan.yaml
@@ -0,0 +1,99 @@
+metadata:
+  name: sa8155p-adp-release-2.0-test-plan
+  description: Qualcomm SA8155p-ADP IoT platform Test Plan
+  os: openembedded
+  devices:
+    - adp
+  maintainer:
+    - bhupesh.sharma@linaro.org
+    - nicolas.dechesne@linaro.org
+  approver:
+    - bhupesh.sharma@linaro.org
+  owner:
+    - Bhupesh Sharma
+  format: Linaro Test Plan v2
+  version: 0.1
+
+tests:
+  automated:
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/smoke/smoke.yaml
+      parameters:
+              TESTS: pwd, uname -a, ip a, vmstat, lsblk
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/usb-smoke/usb-smoke-test.yaml
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/toolchain-smoke/toolchain-smoke.yaml
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/device-tree/device-tree.yaml
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/meminfo/meminfo.yaml
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/dd-wr-speed/dd-wr-speed.yaml
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/sysbench/sysbench.yaml
+      parameters:
+              NUM_THREADS: 4
+              TESTS: cpu memory threads mutex
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/device-read-perf/device-read-perf.yaml
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/hackbench/hackbench.yaml
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/linpack/linpack.yaml
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/openssl/openssl-speed.yaml
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/stream/stream-uniprocessor.yaml
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/pi-stress/pi-stress.yaml
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/pm-qa/pm-qa.yaml
+      parameters:
+        TESTS: cpufreq cpuidle cpuhotplug cputopology
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/docker/docker.yaml
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/bootrr/bootrr.yaml
+      parameters:
+              BOARD: qcom,sa8155p-adp
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/ptest/ptest.yaml
+      params:
+        EXCLUDE: bluez5 libxml2 parted python strace
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/crypto/crypto.yaml
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/crypto/cryptsetup.yaml
+    - repository: https://github.com/Linaro/test-definitions
+      from: git
+      path: automated/linux/iozone/iozone.yaml
+
+  manual:
+    - path: manual/generic/serial-console.yaml
+      repository: https://github.com/Linaro/test-definitions
+    - path: manual/generic/linux/usb-keyboard.yaml
+      repository: https://github.com/Linaro/test-definitions
+    - path: manual/generic/linux/usb-storage.yaml
+      repository: https://github.com/Linaro/test-definitions
+    - path: manual/generic/linux/usb-camera.yaml
+      repository: https://github.com/Linaro/test-definitions
diff --git a/openembedded/boards/sa8155p-adp/Testreport.textile b/openembedded/boards/sa8155p-adp/Testreport.textile
new file mode 100644
index 0000000..304599c
--- /dev/null
+++ b/openembedded/boards/sa8155p-adp/Testreport.textile
@@ -0,0 +1,92 @@
+{% set base_url = [server, '/%s/%s/' % (group.slug, project.slug)]|join %}
+h1{color:blue;}. Test Report for Linaro OpenEmbedded RPB release for Qualcomm(tm) SA8155p-ADP IoT platform
+
+!{width: 20%; padding: 5px; }https://www.linaro.org/assets/images/Linaro-Logo.svg(Linaro Logo)!
+
+*Date:* {{ metadata.now }}
+
+<hr>
+
+h1. Build information
+
+This test report is generated from the test data available on the following build location:
+
+"{{ base_url + 'build/' + build.version + '/' }}":{{ base_url + 'build/' + build.version + '/' }}
+
+For this release, since the SA8155p-ADP platform is not available in the Linaro main LAVA lab, the tests are executed locally, using LAVA test-runner utility.
+
+h1. Test Summary
+
+{% if testruns.values()|selectattr('job_url')|list|length %}
+The following LAVA jobs were executed:
+
+table{table-layout: fixed; width: 50%; border:1px solid black; border-spacing: 10px; text-align: left; border-collapse:collapse}.
+|:. 100| 20| 60| 20|
+{background:#ddd}. |_. Job ID |_. Job URL |_. Job Status |
+{%- for tr in testruns.values() %}
+| {{ tr.job_id }} | "{{ tr.job_url }}":{{ tr.job_url }} | {{ tr.job_status }} |
+{%- endfor %}
+{% endif %}
+
+{%- set pass= tests_automated.values()|selectattr('status', 'equalto', 'pass')|list|length %}
+{%- set fail= tests_automated.values()|selectattr('status', 'equalto', 'fail')|list|length %}
+{%- set skip= tests_automated.values()|selectattr('status', 'equalto', 'skip')|list|length %}
+{%- set benchmarks= metrics.values()|list|length %}
+{%- set automated = pass+fail+skip+benchmarks %}
+
+{% if automated > 0 %}
+table{width: 25%; border:1px solid black; border-spacing: 10px; text-align: left; border-collapse:collapse}.
+{background:#ddd}. |_. Pass |_. Fail |_. Skip |_. Benchmarks|_. Total |
+| {{pass}} | {{fail}} | {{skip}} | {{benchmarks}} | {{automated}} |
+{% else %}
+None.
+{% endif %}
+
+h1. Test Results
+
+{% if automated %}
+{% for suite, _tests in tests_automated.values()|groupby('suite') %}
+h3. {{ loop.index }}. {{ suites.values() | selectattr('url', 'equalto', suite) | first | attr('slug')}}
+
+_Summary_
+{%- set pass= _tests|selectattr('status', 'equalto', 'pass')|list|length %}
+{%- set fail= _tests|selectattr('status', 'equalto', 'fail')|list|length %}
+{%- set skip= _tests|selectattr('status', 'equalto', 'skip')|list|length %}
+
+table{width: 10%; border:1px solid black; border-spacing: 10px; text-align: left; border-collapse:collapse}.
+{background:#ddd}. |_. Result |_. |
+ | Pass  | {{pass}} |
+ | Fail  | {{fail}} |
+ | Skip  | {{skip}} |
+{background:#ddd}.  | Total | {{ pass + fail + skip}} |
+
+_Tests details_
+
+table{table-layout: fixed; width: 50%; border:1px solid black; border-spacing: 10px; text-align: left; border-collapse:collapse}.
+|:. 100| 80| 20|
+{background:#ddd}. |_. Test Case |_. Result |
+    {%- for test in _tests %}
+        | {{ test.short_name }} | {{ test.status }} |
+    {%- endfor %}
+{% endfor %}
+{% else %}
+None.
+{% endif %}
+
+h1. Benchmarks Results
+
+{% if metrics.values()|list|length %}
+{% for suite, _tests in metrics.values()|groupby('suite') %}
+h3. {{ loop.index }}. {{ suites.values() | selectattr('url', 'equalto', suite) | first | attr('slug')}}
+
+table{table-layout: fixed; width: 50%; border:1px solid black; border-spacing: 10px; text-align: left; border-collapse:collapse}.
+|:. 100| 80| 20|
+{background:#ddd}. |_. Test Case |_. Result |
+    {%- for test in _tests %}
+        | {{ test.short_name }} | {{ test.result }} |
+    {%- endfor %}
+
+{% endfor %}
+{% else %}
+None.
+{% endif %}