kci_build: add pull_tarball subcommand

kci_build pull_tarball downloads kernel sources tarball from the
specified url and untars it to the kdir. It was impelemented to replace
the Jenkins pipeline implementation of this mechanism.

Signed-off-by: Michal Galka <michal.galka@collabora.com>

3 files changed, 58 insertions, 0 deletions

diff --git a/kci_build b/kci_build
index a08b9f6..d483ebe 100755
--- a/kci_build
+++ b/kci_build
@@ -318,6 +318,16 @@ Invalid arguments, please provide at least one of these sets of options:
             json_path=args.json_path)
 
 
+class cmd_pull_tarball(Command):
+    help = "Downloads and untars kernel sources"
+    args = [Args.kdir, Args.url]
+    opt_args = [Args.filename, Args.retries]
+
+    def __call__(self, configs, args):
+        return kernelci.build.pull_tarball(args.kdir, args.url,
+                                           args.filename, args.retries)
+
+
 if __name__ == '__main__':
     args = parse_args("kci_build", "build-configs.yaml", globals())
     configs = kernelci.config.build.from_yaml(args.yaml_configs)
diff --git a/kernelci/build.py b/kernelci/build.py
index 7ed0374..30bdee3 100644
--- a/kernelci/build.py
+++ b/kernelci/build.py
@@ -23,6 +23,7 @@ import requests
 import shutil
 import stat
 import subprocess
+import tarfile
 import tempfile
 import time
 import urlparse
@@ -341,6 +342,22 @@ def push_tarball(config, kdir, storage, api, token):
     return tarball_url
 
 
+def pull_tarball(kdir, url, dest_filename, retries):
+    if os.path.exists(kdir):
+        shutil.rmtree(kdir)
+    os.makedirs(kdir)
+    for i in range(1, retries + 1):
+        if _download_file(url, dest_filename):
+            break
+        if i < retries:
+            time.sleep(2 ** i)
+    else:
+        return False
+    with tarfile.open(dest_filename, 'r:*') as tarball:
+        tarball.extractall(kdir)
+    return True
+
+
 def _add_frag_configs(kdir, frag_list, frag_paths, frag_configs):
     for frag in frag_list:
         if os.path.exists(os.path.join(kdir, frag.path)):
@@ -872,3 +889,14 @@ def publish_kernel(kdir, install='_install_', api=None, token=None,
         resp.raise_for_status()
 
     return True
+
+
+def _download_file(url, dest_filename, chunk_size=1024):
+    resp = requests.get(url, stream=True)
+    if resp.status_code == 200:
+        with open(dest_filename, 'wb') as out_file:
+            for chunk in resp.iter_content(chunk_size):
+                out_file.write(chunk)
+        return True
+    else:
+        return False
diff --git a/kernelci/cli.py b/kernelci/cli.py
index 6028309..ab340e2 100644
--- a/kernelci/cli.py
+++ b/kernelci/cli.py
@@ -190,6 +190,26 @@ class Args(object):
         'help': "Test plan name",
     }
 
+    url = {
+        'name': '--url',
+        'help': "Kernel sources download URL",
+    }
+
+    filename = {
+        'name': '--filename',
+        'help': "Kernel sources destination filename",
+        'required': False,
+        'default': 'linux-src.tar.gz',
+    }
+
+    retries = {
+        'name': '--retries',
+        'help': 'Number of retries before download fails',
+        'required': False,
+        'default': 1,
+        'type': int,
+    }
+
 
 class Command(object):
     help = None