trigger-lt-qcom-linux-{integration,mainline,release}: poll kernelci using API

Instead of polling on storage.kernelci.org which has proven to be
error prone, let's use the kernelci.org API to check for builds.

The script kernelci.py checks if new builds are available, and it
returns 1 build at max. If more than 1 build are available it should
be called several times.

Rename get_latest_kernel_build.py to linaroci.py now only handles Linaro
CI kernel builds, update release trigger.

Change-Id: Ib45979b9e5065b7dd85abd20ae130f5f2252fef3
Signed-off-by: Nicolas Dechesne <nicolas.dechesne@linaro.org>

2 files changed, 59 insertions, 42 deletions

diff --git a/lt-qcom-linux-test/kernelci.py b/lt-qcom-linux-test/kernelci.py
new file mode 100755
index 00000000..f9eeecbe
--- /dev/null
+++ b/lt-qcom-linux-test/kernelci.py
@@ -0,0 +1,58 @@
+#!/usr/bin/python3
+
+import os
+import sys
+import requests
+import json
+
+from urllib.parse import urljoin
+
+BACKEND_URL = "https://api.kernelci.org"
+
+def main(token, job, arch, defconfig, output='output.log'):
+    headers = {
+        "Authorization": token
+    }
+
+    params = {
+        "job": job,
+        "arch": arch,
+        "defconfig_full": defconfig,
+        "date_range": 2,
+        "status": "PASS",
+    }
+
+    url = urljoin(BACKEND_URL, "/builds")
+    req = requests.get(url, params=params, headers=headers)
+    if req.status_code != 200:
+        raise Exception("Unable to download %s" % url)
+
+    resp = req.json()
+
+    os.makedirs(".builds", exist_ok=True)
+    for build in resp['result']:
+        status = '.builds/' + '_'.join([build[key] for key in ['job', 'git_describe', 'arch', 'defconfig_full']])
+        if os.access(status, os.R_OK):
+            continue
+
+        workdir = '/'.join([build[key] for key in ['job', 'git_describe', 'arch', 'defconfig_full']])
+        with open(status, 'w') as f:
+            os.utime(status)
+
+        print("Found something!")
+        res_url = "https://storage.kernelci.org/" + build['file_server_resource'] + "/"
+        print(res_url)
+        with open(output, 'w') as f:
+            f.write("KERNEL_IMAGE_URL=" + res_url + build['kernel_image'] + '\n')
+            f.write("KERNEL_MODULES_URL=" + res_url + build['modules'] + '\n')
+            f.write("KERNEL_VERSION=" + build['git_describe'] + '\n')
+            f.write("KERNEL_DT_URL=" + res_url + build['dtb_dir'] + '\n')
+
+        # let's quit, process one new job at max
+        sys.exit(0)
+
+    # nothing to do
+    sys.exit(1)
+
+if __name__ == "__main__":
+    main(*sys.argv[1:])
diff --git a/lt-qcom-linux-test/get_latest_kernel_build.py b/lt-qcom-linux-test/linaroci.py
index f30e884a..a4cc705d 100644..100755
--- a/lt-qcom-linux-test/get_latest_kernel_build.py
+++ b/lt-qcom-linux-test/linaroci.py
@@ -9,33 +9,6 @@ import dateutil.parser
 
 from bs4 import BeautifulSoup, SoupStrainer
 
-
-def get_kernel_ci_build(url, arch_config):
-    f = urllib2.urlopen(url)
-    page = f.read()
-    soup = BeautifulSoup(page, "html.parser")
-
-    last_build = -1
-    for tr in soup.select('table > tbody > tr'):
-        if 'Parent directory' in tr.text or 'last.commit' in tr.text or '-lava-bisect-' in tr.text:
-            continue
-
-        if last_build == -1:
-            last_build = tr
-        elif dateutil.parser.parse(tr.contents[2].text) > \
-                dateutil.parser.parse(last_build.contents[2].text):
-            last_build = tr
-
-    url = url + last_build.contents[0].text + arch_config
-
-    image_url = url + 'Image'
-    dt_url = url + "dtbs"
-    modules_url = url + 'modules.tar.xz'
-    version = last_build.contents[0].text[0:-1] # remove last / char
-
-    return (image_url, dt_url, modules_url, version)
-
-
 def _find_last_build_in_linaro_ci(url):
     last_build = -1
     rex = re.compile('(?P<last_build>\d+)')
@@ -122,13 +95,6 @@ def validate_if_already_built(url, artifacts_urls):
 
 
 def main():
-    kernel_build_type = os.environ.get('KERNEL_BUILD_TYPE', 'KERNEL_CI')
-
-    kernel_ci_base_url = os.environ.get('KERNEL_CI_BASE_URL',
-                                        'https://storage.kernelci.org/qcom-lt/integration-linux-qcomlt/')
-    kernel_ci_arch_config = os.environ.get('KERNEL_CI_ARCH_CONFIG',
-                                           'arm64/defconfig/gcc-8/')
-
     linaro_ci_base_url = os.environ.get('LINARO_CI_BASE_URL',
                                         'https://snapshots.linaro.org/member-builds/qcomlt/kernel/')
 
@@ -142,14 +108,7 @@ def main():
     modules_url = None
     version = None
 
-    if kernel_build_type == 'KERNEL_CI':
-        (image_url, dt_url, modules_url, version) = get_kernel_ci_build(kernel_ci_base_url,
-                                                                        kernel_ci_arch_config)
-    elif kernel_build_type == 'LINARO_CI':
-        (image_url, dt_url, modules_url, version) = get_linaro_ci_build(linaro_ci_base_url)
-    else:
-        print('ERROR: Kernel build type (%s) isn\'t supported' % kernel_build_type)
-        sys.exit(1)
+    (image_url, dt_url, modules_url, version) = get_linaro_ci_build(linaro_ci_base_url)
 
     print("KERNEL_DT_URL=%s" % dt_url)