diff options
author | Andy Doan <andy.doan@linaro.org> | 2015-08-24 12:39:11 -0500 |
---|---|---|
committer | Andy Doan <andy.doan@linaro.org> | 2015-08-24 12:40:48 -0500 |
commit | be1eac5f2c052c3bc6eb0fd4c2ab43b8674cdb23 (patch) | |
tree | 553eddfb97a813fc702573f06a2e4dfdedaca009 | |
parent | a22bfb931095605cc202fdaf432cd01a3af43eac (diff) |
add support for new v3 api
l-l-p has a new v3 API that takes advantage of Amazon S3. Specifically
it generates temp urls (or pre-signed urls) that push the actual file
to.
Change-Id: I867c7ab7565db586f00e96c83417675036bcc56e
-rwxr-xr-x | linaro-cp.py | 63 |
1 files changed, 58 insertions, 5 deletions
diff --git a/linaro-cp.py b/linaro-cp.py index ed5fc10..5819211 100755 --- a/linaro-cp.py +++ b/linaro-cp.py @@ -4,6 +4,7 @@ import argparse import atexit import cStringIO import os +import mimetypes import pycurl import sys import tempfile @@ -142,11 +143,65 @@ class API_v2(API_v1): return self._upload_data(url, [('foo', 'bar')], headers) +class API_v3(API_v1): + def __init__(self, server, build_info, api_key): + super(API_v3, self).__init__(server, build_info, api_key) + self.api_base = server + '/api/v3/publish/' + + def _upload_data(self, url, data, headers=None, retry_count=3): + self.last_headers = cStringIO.StringIO() + self.curl.setopt(pycurl.HEADERFUNCTION, self.last_headers.write) + return super(API_v3, self)._upload_data( + url, data, headers, retry_count) + + def _put_s3(self, url, filename, mtype): + response = cStringIO.StringIO() + size = os.path.getsize(filename) + headers = ['Content-Type: ' + mtype] + c = pycurl.Curl() + c.setopt(pycurl.URL, url) + c.setopt(pycurl.HTTPHEADER, headers) + c.setopt(pycurl.INFILESIZE, size) + c.setopt(pycurl.PUT, 1) + c.setopt(pycurl.WRITEFUNCTION, response.write) + with open(filename, 'rb') as f: + c.setopt(pycurl.INFILE, f) + c.perform() + code = c.getinfo(pycurl.RESPONSE_CODE) + if code not in (200, 201): + return response.getvalue() + + def upload_file(self, url, filename): + # ask llp for an s3 tempurl: + mtype = mimetypes.guess_type(filename)[0] + if not mtype: + mtype = 'other' + + headers = ['AuthToken: ' + self.api_key] + code = self._upload_data(url, [('Content-Type', mtype)], headers) + if code: + return code + + # now find the tempurl and really publish + for header in self.last_headers.getvalue().split('\n'): + if header.startswith('Location:'): + location = header[9:].strip() + return self._put_s3(location, filename, mtype) + raise RuntimeError('l-l-p response missing s3 location') + + def link_latest(self, dst): + headers = ['AuthToken: ' + self.api_key] + url = self.server + '/api/v3/link_latest/' + dst + # pycurl requires data to be passed, or it will do an + # HTTP GET even though we said to POST + return self._upload_data(url, [('foo', 'bar')], headers) + + def main(): parser = argparse.ArgumentParser( description='Copy file(s) from source to destination') parser.add_argument('-k', '--key', help='key used for the copy') - parser.add_argument('-a', '--api_version', choices=('1', '2'), default='1', + parser.add_argument('-a', '--api_version', choices=('1', '2', '3'), default='1', help='API version to use. default=%(default)s') parser.add_argument('--server', default='http://snapshots.linaro.org/', help='Publishing API server. default=%(default)s') @@ -191,10 +246,8 @@ def main(): atexit.register(os.unlink, arguments.build_info) os.write(fd, build_info) - if arguments.api_version == '1': - api = API_v1(arguments.server, arguments.build_info, key) - else: - api = API_v2(arguments.server, arguments.build_info, key) + cls = globals()['API_v' + arguments.api_version] + api = cls(arguments.server, arguments.build_info, key) if arguments.split_job_owner: # Rewrite job path .../owner_jobname/123 -> .../~owner/jobname/123 , |