path: root/art_post_script.py

#!/usr/bin/python

import xmlrpclib
import json
import re
import os
import ast
import zipfile
import base64
import shutil
import argparse
import logging
import requests
import pickle
from argparse import RawTextHelpFormatter
from StringIO import StringIO
import xml.etree.ElementTree as ET
from jenkinsapi.jenkins import Jenkins
from urlparse import urljoin, urlsplit

CTS_RESULT_FILE_NAME = "testResult.xml"
logging.basicConfig(format='%(levelname)s:  %(message)s', level=logging.INFO)
logger = logging.getLogger(__name__)
#logger.setLevel(logging.INFO)

MANIFEST_ENDPOINT           = "manifest/"
RESULT_ENDPOINT             = "result/"
RESULTDATA_ENDPOINT         = "resultdata/"
BRANCH_ENDPOINT             = "branch/"
BOARD_ENDPOINT              = "board/"
BOARDCONFIGURATION_ENDPOINT = "boardconfiguration/"
BENCHMARK_ENDPOINT          = "benchmark/"

class InfoRequestError(Exception):
    def __init__(self, reason):
        self.reason = reason

    def __str__(self):
        return repr(self.reason)


class ArtDb(object):
    def __init__(self, backend_url, username, password):
        self.backend_url = backend_url
        self.auth_user   = username
        self.auth_pw     = password

    def retrieve_object_list(backend_url, endpoint, params):
        url = urljoin(backend_url, endpoint)
        #response = requests.get(url, params=params, headers=headers)
        response = requests.get(url, params=params)
        if response.status_code == 200:
            return response.json()
        else:
            print response.status_code
            print response.reason
            print response.text
        return []
    
    def push_object(self, endpoint, params):
        url = urljoin(self.backend_url, endpoint)
        #headers.update({'Content-Type':'application/json'})
        #response = requests.post(url, data=params, headers=headers)
        response = requests.post(url, data=params)
        #pprint.pprint(params)
        if response.status_code < 300:
            return response.json()
        else:
            print response.status_code
            print response.text
        return []

    def submit_result_to_db(self, data):
#        data = { "build_url"     : "", 
#                 "manifest"      : "",
#                 "gerrit_params" : {},
#                 "test_result"   : [] }
        
        params = {
            "manifest": data['manifest']
        }
        manifest = self.push_object(MANIFEST_ENDPOINT, params)
        params = {
            "name": data['test_result']['benchmark_name']
        }
        benchmark = self.push_object(BENCHMARK_ENDPOINT, params)
        params = {
            "name" : data['test_result']['board_config']
        }
        boardconfig = self.push_object(BOARDCONFIGURATION_ENDPOINT, params)
        params = {
            "displayname"   : data['test_result']['board'],
            "display"       : data['test_result']['board'],
            "configuration" : boardconfig['id']
        }
        board = self.push_object(BOARD_ENDPOINT, params)
        params = {
            "name" : data['gerrit_params']['GERRIT_BRANCH']
        }
        branch = self.push_object(BRANCH_ENDPOINT, params)
        params = {
            "board":                    board['id'],
            "branch":                   branch['id'],
            "manifest":                 manifest['id'],
            "build_url":                data['build_url'],
            "gerrit_change_id":         data['gerrit_params']['GERRIT_CHANGE_ID'],
            "gerrit_patchset_number":   data['gerrit_params']['GERRIT_PATCHSET_NUMBER'],
            "gerrit_change_number":     data['gerrit_params']['GERRIT_CHANGE_NUMBER'],
            "gerrit_change_url":        data['gerrit_params']['GERRIT_CHANGE_URL']
        }
        result = self.push_object(RESULT_ENDPOINT, params)
        params = {
            "benchmark": benchmark['id'],
            "result": result['id']
        }
        for t in data['test_result']['subscore']:
            params.update(t) 
#            logger.info("Submit: %s" % str(params))
            self.push_object(RESULTDATA_ENDPOINT, params)


class LinaroAndroidBuildSystem(object):
    def __init__(self, base_url, build_job,
                 username=None, password=None):
        logger.info("Jenkins build job: %s" %  build_job)
        self.base_url  = base_url
        self.build_job = build_job
        self.username  = username
        self.password  = password
        try:
            self.jenkins    = Jenkins(self.base_url, self.username, self.password)
            self.project    = self.jenkins[self.build_job]
        except:
            logger.error("Can not get Jenkins job: %s" % self.build_job)
            raise

    def get_build(self, build_no):
        try:
            build = self.project.get_build(build_no)
            return build
        except:
            logger.error("Can not get build: #%d" % build_no)
            raise

    def get_build_test_job_ids(self, build_no):
        logger.info("Querying build #%d test job ids from Jenkins build job: %s" % 
              (build_no, self.build_job))
        job_id_list = []
        build = self.get_build(build_no)
        job_id_list = re.findall("LAVA Job Id:\s\[?\'?(?P<master_job_id>[\d\.]+)", build.get_console())

        return job_id_list

    def get_build_manifest(self, build_no):
        manifest_file_name = "pinned-manifest.xml"
        rest_url = "%s/job/%s/%d/artifact/%s" % (self.base_url, self.build_job, build_no, manifest_file_name)
        manifest = requests.get(rest_url, auth=(self.username, self.password))
        if manifest.status_code != 200:
            raise InfoRequestError("[%s] %s" % (manifest.status_code, manifest.reason))
        else:
            return manifest.content

    def get_build_gerrit_parameters(self, build_no):
        gerrit_param = { "GERRIT_BRANCH": "",
                         "GERRIT_CHANGE_ID": "",
                         "GERRIT_PATCHSET_NUMBER": "",
                         "GERRIT_CHANGE_NUMBER": "",
                         "GERRIT_CHANGE_URL": "" }
        rest_url = "%s/job/%s/%d/api/json" % (self.base_url, self.build_job, build_no)
        req = requests.get(rest_url, auth=(self.username, self.password))
        if req.status_code != 200:
            raise InfoRequestError("[%s] %s" % (manifest.status_code, manifest.reason))
        else:
            build_info = req.json()
        try:
            params   = (a['parameters'] for a in build_info['actions'] if 'parameters' in a).next()
        except:
            logger.error("Can not get build: #%d parameters." % build_no)
            raise
        for p in params:
            if p['name'] in gerrit_param:
                gerrit_param[p['name']] = p['value']

        if not ''.join(gerrit_param.values()):
            logger.error("Can not get gerrit parameters in this build!")
            raise InfoRequestError("Can not find any gerrir related parameters in this build")

        return gerrit_param


class LAVA(object):
    def __init__(self, username, token, lava_server="https://validation.linaro.org/"):
        try:
            self.lava_server = lava_server
            self.username    = username
            self.token       = token
            server_url       = "https://%s:%s@%sRPC2/" % (self.username, self.token, self.lava_server.split("//")[1])
            self.server      = xmlrpclib.ServerProxy(server_url)
        except:
            logger.warning("Can not connect to LAVA server: %s" % lava_server)
            raise

    def get_test_results(self, job_no):
        test_result_list = []
        test_result = { "board": "",
                        "board_config": "",
                        "benchmark_name": "",
                        "subscore": [] }
        try:
            job_status = self.server.scheduler.job_status(job_no)
        except:
            logger.warning("Can not get any information for test job: %s" % job_no)
            raise

        if job_status['job_status'] != 'Complete':
            logger.warning("!!Job #%s is not completed.\n\tJob status: %s!! " % (job_no, job_status['job_status']))
        else:
            try:
                result_bundle = json.loads(self.server.dashboard.get(job_status['bundle_sha1'])['content'])
                target = (t for t in result_bundle['test_runs'] if t['test_id'] == 'multinode-target' or
                          t['test_id'] == 'lava-android-benchmark-target').next()
                host   = (t for t in result_bundle['test_runs'] if t['test_id'] == 'art-microbenchmarks' or
                          t['test_id'] == 'lava-android-benchmark-host').next()
                src = (s for s in host['software_context']['sources'] if 'test_params' in s).next()
            except:
                logger.warning("Job #%s seems not be a complete job, can not get test result" % job_no)
                raise
            # Get target device name
            test_result['board']        = target['attributes']['target']
            test_result['board_config'] = target['attributes']['target']
            # Get test name and test result
            if host['test_id'] == 'lava-android-benchmark-host':
                # This is a 3rd. party benchmark test
                test_result['benchmark_name'] = ast.literal_eval(src['test_params'])['TEST_NAME']
                # Get test results
                for t in host['test_results']:
                    if 'measurement' in t:
                        test_case = { "name": t['test_case_id'],
                                      "measurement" : t['measurement'] }
                        test_result['subscore'].append(test_case)
                test_result_list.append(test_result)
            else:
                # This is an art-microbenchmarks test
                # The test name and test results are in the attachmented pkl file
                # get test results for the attachment
                test_mode        = ast.literal_eval(src['test_params'])['MODE']
                pkl_content      = (a['content'] for a in host['attachments'] if a['pathname'].endswith('pkl')).next()
                test_result_dict = pickle.loads(base64.b64decode(pkl_content))
                # Key Format: benchmarks/micro/<BENCHMARK_NAME>.<SUBSCORE>
                # Extract and unique them to form a benchmark name list
                test_result_keys = list(bn.split('/')[-1].split('.')[0] for bn in test_result_dict.keys())
                benchmark_list   = list(set(test_result_keys))
                for benchmark in benchmark_list:
#                    print("Benchmark: %s" % benchmark)
                    test_result = {}
                    # benchmark iteration
                    test_result['benchmark_name'] = benchmark
                    test_result['subscore'] = []
                    key_word = "/%s." % benchmark
                    tests    = ((k, test_result_dict[k]) for k in test_result_dict.keys() if k.find(key_word) > 0)
                    for test in tests:
                        # subscore iteration
                        subscore = "%s_%s" % (test[0].split('.')[-1], test_mode)
#                        print("\tSubscore: %s" % subscore)
#                        print("\tValues: %s" % str(test[1]))
                        for i in test[1]:
                            test_case = { "name": subscore,
                                          "measurement": i }
                            test_result['subscore'].append(test_case)

                    test_result_list.append(test_result)

#        if not test_result['test_result']:
#            raise InfoRequestError("No test cases in job #%s" % job_no)

        return test_result_list


if __name__ == '__main__':

    # Replace these authentication credential with environment variables
    # Define them by ourselves
    JENKINS_USERNAME    = "arthur.she@linaro.org"
    JENKINS_PASSWORD    = ""
    LAVA_USERNAME       = "arthur.she"
    LAVA_TOKEN          = ""
    BACKEND_DB_USERNAME = ""
    BACKEND_DB_PASSWORD = ""

    parser          = argparse.ArgumentParser()
    parser.add_argument("-u", "--build-url", dest="build_url", required=True,
                        help="Specify the Jenkins build job url. This is MANDATORY.")
    parser.add_argument("-db", "--backend-db", dest="backend_db", required=True,
                        help="Specify the backend database url. This is MANDATORY.")

    args = parser.parse_args()

    if args.build_url[len(args.build_url) - 1] != '/':
        url = args.build_url + '/'
    else:
        url = args.build_url
    ul = url.split('/')
    jenkins_build_number = ul[-2]
    jenkins_project      = ul[-3]
    jenkins_base_url     = '/'.join(ul[:-4])

    logger.info("build_no: %s\nproj: %s\nbase_url: %s" % (jenkins_build_number, jenkins_project, jenkins_base_url))

    try:
        jenkins    = LinaroAndroidBuildSystem(jenkins_base_url, jenkins_project, JENKINS_USERNAME, JENKINS_PASSWORD)
        lava       = LAVA(LAVA_USERNAME, LAVA_TOKEN)
        backend_db = ArtDb(args.backend_db, BACKEND_DB_USERNAME, BACKEND_DB_PASSWORD)
    except Exception as e:
        logger.error("Initial objects error.\n%s" % e)
        exit(-1)
    try:
        gerrit_param = jenkins.get_build_gerrit_parameters(int(jenkins_build_number))
        manifest     = jenkins.get_build_manifest(int(jenkins_build_number))
    except Exception as e:
        logger.error("Can not get required build information!\n%s" % e)
        exit(-1)

    data = { "build_url" : args.build_url,
             "manifest": manifest,
             "gerrit_params": gerrit_param,
             "test_result": {} }

    test_job_ids = jenkins.get_build_test_job_ids(int(jenkins_build_number))
    logger.info("Test job IDs: %s" % str(test_job_ids))
    for tid in test_job_ids:
        logger.info("Querying job #%s test results" % tid)
        try:
            data['test_result'] = lava.get_test_results(tid)
        except Exception as e:
            logger.warning(e)

        backend_db.submit_result_to_db(data)