build-trigger.jpl: schedule a test-runner job for each build

Schedule a test-runner job to get tests submitted after each kernel
build completes.  This avoids having to wait for all the builds of the
series to have completed before starting tests, and also avoids having
to read build data from the backend as it's already available locally
in Jenkins via meta-data archived in build artifacts.

Store the lab-specific info once in JSON artifact files that
test-runner jobs can then reuse to avoid querying the labs every time.

Add a LABS_WHITELIST optional parameter to only iterate over a subset
of the labs defined in lab-configs.yaml.  This is especially useful on
test instances such as staging.kernelci.org to only submit to certain
labs.

Signed-off-by: Guillaume Tucker <guillaume.tucker@collabora.com>

1 files changed, 104 insertions, 21 deletions

diff --git a/jenkins/build-trigger.jpl b/jenkins/build-trigger.jpl
index 9e23b6f..ebb26ce 100644
--- a/jenkins/build-trigger.jpl
+++ b/jenkins/build-trigger.jpl
@@ -28,6 +28,8 @@ PUBLISH (boolean)
   Publish build results via the KernelCI backend API
 EMAIL (boolean)
   Send build results via email
+LABS_WHITELIST
+  List of labs to include in the tests, all labs will be tested by default.
 KCI_API_URL (https://api.kernelci.org)
   URL of the KernelCI backend API
 KCI_TOKEN_ID
@@ -53,7 +55,7 @@ def configAlreadyBuilt(config, kci_core) {
 
     dir(kci_core) {
         new_commit = sh(
-        script: """
+        script: """\
 ./kci_build \
 check_new_commit \
 --config=${config} \
@@ -143,21 +145,10 @@ list_kernel_configs \
     }
 }
 
-def addBuildOpts(config, kci_core, opts) {
-    dir(kci_core) {
-        opts['config'] = config
-
-        def opts_raw = sh(
-            script: """\
-./kci_build \
-tree_branch \
---config=${config} \
-""", returnStdout: true).trim()
-        def opt_list = opts_raw.tokenize('\n')
-        opts['tree'] = opt_list[0]
-        opts['git_url'] = opt_list[1]
-        opts['branch'] = opt_list[2]
+def listArchitectures(kci_core, config) {
+    def arch_list = []
 
+    dir(kci_core) {
         def raw_variants = sh(
             script: """\
 ./kci_build \
@@ -166,7 +157,6 @@ list_variants \
 """, returnStdout: true).trim()
         def variants = raw_variants.tokenize('\n')
 
-        def arch_list = []
         for (String variant: variants) {
             def raw_variant_arch_list = sh(
                 script: """\
@@ -181,11 +171,50 @@ arch_list \
                 if (!arch_list.contains(arch))
                     arch_list.add(arch)
         }
-        opts['arch_list'] = arch_list
     }
+
+    return arch_list
 }
 
-def buildKernelStep(job, arch, defconfig, build_env, opts) {
+def addBuildOpts(config, kci_core, opts) {
+    dir(kci_core) {
+        opts['config'] = config
+
+        def opts_raw = sh(
+            script: """\
+./kci_build \
+tree_branch \
+--config=${config} \
+""", returnStdout: true).trim()
+        def opt_list = opts_raw.tokenize('\n')
+        opts['tree'] = opt_list[0]
+        opts['git_url'] = opt_list[1]
+        opts['branch'] = opt_list[2]
+    }
+}
+
+def scheduleTests(build_job_name, build_job_number, labs, kci_core) {
+    dir(kci_core) {
+        def labs_str = ""
+        for (lab in labs)
+            labs_str += "${lab} "
+
+        def str_params = [
+            'LABS': labs_str.trim(),
+            'TRIGGER_JOB_NAME': env.JOB_NAME,
+            'TRIGGER_JOB_NUMBER': env.BUILD_NUMBER,
+            'BUILD_JOB_NAME': build_job_name,
+            'BUILD_JOB_NUMBER': "${build_job_number}",
+        ]
+        def params = []
+
+        def j = new Job()
+        j.addStrParams(params, str_params)
+        build(job: 'test-runner', parameters: params, propagate: false)
+    }
+}
+
+def buildKernelStep(job, arch, defconfig, build_env, opts, labs, kci_core) {
     def str_params = [
         'ARCH': arch,
         'DEFCONFIG': defconfig,
@@ -201,7 +230,12 @@ def buildKernelStep(job, arch, defconfig, build_env, opts) {
     def j = new Job()
     j.addStrParams(job_params, str_params)
 
-    return { build(job: job, parameters: job_params, propagate: false) }
+    return {
+        def res = build(job: job, parameters: job_params, propagate: false)
+        print("${res.number}: ${arch} ${defconfig} ${build_env} ${res.result}")
+        if (res.result == "SUCCESS")
+            scheduleTests(job, res.number, labs, kci_core)
+    }
 }
 
 def buildsComplete(job, opts, arch) {
@@ -229,6 +263,7 @@ node("docker && build-trigger") {
     def kci_core = "${env.WORKSPACE}/kernelci-core"
     def kdir = "${env.WORKSPACE}/configs/${params.BUILD_CONFIG}"
     def mirror = "${env.WORKSPACE}/linux.git"
+    def labs_info = "${env.WORKSPACE}/labs"
     def docker_image = "${params.DOCKER_BASE}base"
     def opts = [:]
     def configs = []
@@ -238,11 +273,57 @@ node("docker && build-trigger") {
     Container: ${docker_image}""")
 
     j.dockerPullWithRetry(docker_image).inside() {
+        def labs = []
+
         stage("Init") {
             timeout(time: 15, unit: 'MINUTES') {
                 j.cloneKciCore(
                     kci_core, params.KCI_CORE_URL, params.KCI_CORE_BRANCH)
             }
+
+            sh(script: "rm -rf ${labs_info}; mkdir -p ${labs_info}")
+
+            dir(kci_core) {
+                def raw_lab_names = sh(
+                    script: "./kci_test list_labs", returnStdout: true).trim()
+                def all_lab_names = raw_lab_names.tokenize('\n')
+                def labs_list = []
+
+                if (params.LABS_WHITELIST) {
+                    def whitelist = params.LABS_WHITELIST.tokenize(' ')
+
+                    for (lab in all_lab_names)
+                        if (whitelist.contains(lab))
+                            labs_list.add(lab)
+                } else {
+                    labs_list = all_lab_names
+                }
+
+                for (lab in labs_list) {
+                    def lab_json = "${labs_info}/${lab}.json"
+                    def token = "${lab}-lava-api"
+                    try {
+                        withCredentials([string(credentialsId: token,
+                                                variable: 'SECRET')]) {
+                            sh(script: """\
+./kci_test \
+get_lab_info \
+--lab=${lab} \
+--lab-json=${lab_json} \
+--user=kernel-ci \
+--token=${SECRET} \
+""")
+                        }
+                        labs.add(lab)
+                    } catch (error) {
+                        print("Error with ${lab}: ${error}")
+                    }
+                }
+            }
+
+            dir(labs_info) {
+                archiveArtifacts("*.json")
+            }
         }
 
         if (params.ALLOW_REBUILD != true) {
@@ -276,7 +357,8 @@ node("docker && build-trigger") {
                 print(step_name)
 
                 builds[step_name] = buildKernelStep(
-                    "kernel-build", arch, defconfig, build_env, opts)
+                    "kernel-build", arch, defconfig, build_env, opts, labs,
+                    kci_core)
 
                 i += 1
             }
@@ -286,7 +368,8 @@ node("docker && build-trigger") {
 
         stage("Complete") {
             /* ToDo: convert kernel-arch-complete as a stage in this job */
-            for (String arch: opts['arch_list']) {
+            def arch_list = listArchitectures(kci_core, params.BUILD_CONFIG)
+            for (String arch: arch_list) {
                 buildsComplete("kernel-arch-complete", opts, arch)
             }
         }