path: root/zorg/jenkins/build.py

"""Build and test clangs."""

import sys
import logging
import os
import subprocess
import datetime
import argparse
import urllib
import shutil

SERVER = "labmaster2.local"


def readme_name(repo):
    """Given a repo, return the name of the readme file."""
    if repo == "libcxx":
        return "LICENSE.TXT"
    return "README.txt"


def next_section(name):
    """Jenkins is setup to parse @@@ xyz @@@ as a new section of the buildlog
    with title xyz.  The section ends with @@@@@@ """
    footer()
    header(name)


def header(name):
    print "@@@", name, "@@@"


def footer():
    print "@@@@@@"


def quote_sh_string(string):
    """Make things that we print shell safe for copy and paste."""
    return "\\'".join("'" + p + "'" for p in string.split("'"))


class Configuration(object):
    """docstring for Configuration"""
    def __init__(self, args):
        super(Configuration, self).__init__()
        self._args = args
        self.workspace = os.environ.get('WORKSPACE', os.getcwd())
        self._src_dir = os.environ.get('SRC_DIR', 'llvm')
        self._lldb_src_dir = os.environ.get('LLDB_SRC_DIR', 'lldb')
        self._build_dir = os.environ.get('BUILD_DIR', 'clang-build')
        self._lldb_build_dir = os.environ.get('LLDB_BUILD_DIR', 'lldb-build')
        self._install_dir = os.environ.get('BUILD_DIR', 'clang-install')
        self.j_level = os.environ.get('J_LEVEL', None)
        self.max_parallel_tests = os.environ.get('MAX_PARALLEL_TESTS', None)
        self.max_parallel_links = os.environ.get('MAX_PARALLEL_LINKS', None)
        self.host_compiler_url = os.environ.get('HOST_URL',
            'http://labmaster2.local/artifacts/')
        self.artifact_url = os.environ.get('ARTIFACT', 'NONE')
        self.job_name = os.environ.get('JOB_NAME', 'NONE')
        self.build_id = os.environ.get('BUILD_ID', 'NONE')
        self.build_number = os.environ.get('BUILD_NUMBER', 'NONE')
        self.svn_rev = os.environ.get('LLVM_REV', 'NONE')

        self.__dict__.update(vars(args))

    def builddir(self):
        """The build output directory for this compile."""
        return os.path.join(self.workspace, self._build_dir)

    def srcdir(self):
        """The derived source directory for this build."""
        return os.path.join(self.workspace, self._src_dir)

    def lldbbuilddir(self):
        """The derived source directory for this lldb build."""
        return os.path.join(self.workspace, self._lldb_build_dir)

    def lldbsrcdir(self):
        """The derived source directory for this lldb build."""
        return os.path.join(self.workspace, self._lldb_src_dir)

    def installdir(self):
        """The install directory for the compile."""
        return os.path.join(self.workspace, self._install_dir)

    def CC(self):
        """Location of the host compiler, if one is present in this build."""
        cc_basedir = os.path.join(self.workspace, 'host-compiler/')
        if os.path.exists(cc_basedir):
            clang_exec_path = os.path.join(cc_basedir, 'bin/clang')
            assert os.path.exists(clang_exec_path), "host-compiler present," \
                " but has no clang executable."
            return clang_exec_path
        else:
            return False

    def liblto(self):
        """Location of the host compiler, if one is present"""
        cc_basedir = os.path.join(self.workspace, 'host-compiler/')
        if os.path.exists(cc_basedir):
            clang_liblto_path = os.path.join(cc_basedir, 'lib/')
            assert os.path.exists(clang_liblto_path), "host-compiler present," \
                " but has no liblto."
            return clang_liblto_path
        else:
            return False

# Global storage for configuration object.
conf = None


def cmake_builder(target):
    """Run a build_type build using cmake and ninja."""
    check_repo_state(conf.workspace)

    env = []
    if conf.lto and conf.liblto():
        dyld_path = conf.liblto()
        env.extend(["env", "DYLD_LIBRARY_PATH=" + dyld_path])

    cmake_cmd = env + ["/usr/local/bin/cmake", "-G", "Ninja",
                       "-DCMAKE_INSTALL_PREFIX=" + conf.installdir(),
                       conf.srcdir()]

    compiler_flags = conf.compiler_flags
    max_parallel_links = conf.max_parallel_links
    if conf.lto:
        compiler_flags += ['-flto']
        cmake_cmd += ['-DLLVM_BUILD_EXAMPLES=Off']
        if not max_parallel_links:
            max_parallel_links = 1
    else:
        cmake_cmd += ['-DLLVM_BUILD_EXAMPLES=On']

    if compiler_flags:
        cmake_cmd += ["-DCMAKE_C_FLAGS={}".format(' '.join(compiler_flags)),
                      "-DCMAKE_CXX_FLAGS={}".format(' '.join(compiler_flags))]

    if max_parallel_links is not None:
        cmake_cmd += ["-DLLVM_PARALLEL_LINK_JOBS={}".format(max_parallel_links)]

    if conf.CC():
        cmake_cmd += ['-DCMAKE_C_COMPILER=' + conf.CC(),
                      '-DCMAKE_CXX_COMPILER=' + conf.CC() + "++"]

    if conf.cmake_build_type:
        cmake_cmd += ["-DCMAKE_BUILD_TYPE=" + conf.cmake_build_type]
    elif conf.debug:
        cmake_cmd += ["-DCMAKE_BUILD_TYPE=Debug"]
    else:
        cmake_cmd += ["-DCMAKE_BUILD_TYPE=Release"]

    for flag in conf.cmake_flags:
        cmake_cmd += [flag]

    if conf.assertions:
        cmake_cmd += ["-DLLVM_ENABLE_ASSERTIONS=On"]
    else:
        cmake_cmd += ["-DLLVM_ENABLE_ASSERTIONS=Off"]

    # Workaround for PR14109: CMake build for compiler-rt should use
    # just-built clang.
    cmake_cmd += ["-DCOMPILER_RT_BUILD_BUILTINS=Off"]

    lit_flags = ['--xunit-xml-output=testresults.xunit.xml', '-v']
    if conf.max_parallel_tests:
        lit_flags += ['-j', conf.max_parallel_tests]
    cmake_cmd += ['-DLLVM_LIT_ARGS={}'.format(' '.join(lit_flags))]

    ninja_cmd = env + ["/usr/local/bin/ninja"]
    if conf.j_level is not None:
        ninja_cmd += ["-j", conf.j_level]

    if target == 'all' or target == 'build':
        header("Cmake")
        run_cmd(conf.builddir(), cmake_cmd)
        footer()
        header("Ninja build")
        run_cmd(conf.builddir(), ninja_cmd)
        footer()

    if target == 'all' or target == 'test':
        header("Ninja test")
        run_cmd(conf.builddir(), ninja_cmd + ['check', 'check-clang'])
        footer()

    if target == 'all' or target == 'testlong':
        header("Ninja test")
        run_cmd(conf.builddir(), ninja_cmd + ['check-all'])
        footer()


def clang_builder(target):
    """Do a configure + make build of clang. Target is the type of build."""
    check_repo_state(conf.workspace)
    configure_cmd = [conf.srcdir() + "/configure"]

    if conf.assertions:
        configure_cmd.append("--enable-assertions")
    else:
        configure_cmd.append("--disable-assertions")

    compiler_flags = conf.compiler_flags
    if conf.lto:
        compiler_flags += ['-flto', '-gline-tables-only']

    if compiler_flags:
        configure_cmd.extend(
            ['--with-extra-options={}'.format(' '.join(compiler_flags))])

    configure_cmd.extend(["--enable-optimized",
        "--disable-bindings", "--enable-targets=x86,x86_64,arm,aarch64",
        "--prefix=" + conf.installdir(), "--enable-libcpp"])
    if conf.CC():
        configure_cmd.append('CC='+conf.CC())
        configure_cmd.append('CXX='+conf.CC()+"++")

    rev_str =  "CLANG_REPOSITORY_STRING={}".format(conf.job_name)
    svn_rev_str = "SVN_REVISION={}".format(conf.svn_rev)
    llvm_rev = "LLVM_VERSION_INFO= ({}: trunk {})".format(
        conf.job_name, conf.svn_rev)

    j_number = "4" if conf.j_level is None else conf.j_level

    make_all = ["make", "-j", j_number, "VERBOSE=1",
                rev_str, svn_rev_str, llvm_rev]

    if conf.lto and conf.liblto():
        dyld_path = conf.liblto()
        make_all.append("DYLD_LIBRARY_PATH=" + dyld_path)

    make_install = ["make", "install-clang", "-j", j_number]

    make_check = ["make", "VERBOSE=1", "check-all",
        'LIT_ARGS=--xunit-xml-output=testresults.xunit.xml -v']

    if target == 'build' or target == 'all':
        header("Configure")
        run_cmd(conf.builddir(), configure_cmd)
        footer()
        header("Make All")
        run_cmd(conf.builddir(), make_all)
        footer()
        header("Make Install")
        run_cmd(conf.builddir(), make_install)
        footer()
        header("Upload artifact")
        build_upload_artifact()
        footer()

    if target == 'test' or target == 'all':
        header("Make Check")
        run_cmd(conf.builddir(), make_check)
        footer()
        # run_cmd(conf.builddir(), make_check_debug)

def lldb_builder():
    """Do an Xcode build of lldb."""

    # Wipe the build folder

    header("Clean LLDB build directory")
    if os.path.exists(conf.lldbbuilddir()):
        shutil.rmtree(conf.lldbbuilddir())
    footer()

    # Build into the build folder

    xcodebuild_cmd = [
        "xcodebuild",
        "-arch", "x86_64",
        "-configuration", "Debug",
        "-scheme", "desktop",
        "-derivedDataPath", conf.lldbbuilddir(),
        "DEBUGSERVER_USE_FROM_SYSTEM=1"]

    header("Build Xcode desktop scheme")
    run_cmd("lldb", xcodebuild_cmd)
    footer()

    # Run C++ test suite (gtests)

    xcodebuild_cmd = [
        "xcodebuild",
        "-arch", "x86_64",
        "-configuration", "Debug",
        "-scheme", "lldb-gtest",
        "-derivedDataPath", conf.lldbbuilddir(),
        "DEBUGSERVER_USE_FROM_SYSTEM=1"]

    header("Build Xcode lldb-gtest scheme")
    run_cmd("lldb", xcodebuild_cmd)
    footer()

    # Run LLDB Python test suite

    xcodebuild_cmd = [
        "xcodebuild",
        "-arch", "x86_64",
        "-configuration", "Debug",
        "-scheme", "lldb-python-test-suite",
        "-derivedDataPath", conf.lldbbuilddir(),
        "DEBUGSERVER_USE_FROM_SYSTEM=1"]

    header("Build Xcode lldb-python-test-suite target")
    # For the unit tests, we don't want to stop the build if there are
    # build errors.  We allow the JUnit/xUnit parser to pick this up.
    run_cmd_errors_okay("lldb", xcodebuild_cmd)
    footer()

def static_analyzer_benchmarks_builder():
  """Run static analyzer benchmarks"""
  header("Static Analyzer Benchmarks")

  benchmark_script = conf.workspace + "/utils-analyzer/SATestBuild.py"
  benchmarks_dir = conf.workspace + "/test-suite-ClangAnalyzer/"

  compiler_bin_dir = conf.workspace + "/host-compiler/bin/"
  scanbuild_bin_dir = conf.workspace + "/tools-scan-build/bin/"

  old_path = os.environ.get("PATH", "")
  env = dict(os.environ, PATH=compiler_bin_dir + os.pathsep +
                              scanbuild_bin_dir + os.pathsep +
                              old_path)

  benchmark_cmd = [benchmark_script,
                   "--strictness", "2"
                  ]
  run_cmd(benchmarks_dir, benchmark_cmd, env=env)

  footer()

def check_repo_state(path):
    """Check the SVN repo at the path has all the
    nessessary repos checked out.  Check this by
    looking for the README.txt in each repo. """

    if os.environ.get('TESTING', False):
        return

    logging.info("Detecting repos in {}".format(path))
    for r in ['llvm', 'clang', 'clang-tools-extra', 'debuginfo-tests', \
            'compiler-rt', 'libcxx', 'debuginfo-tests']:
        detected_path = derived_path('llvm', tree_path(tree='llvm', repo=r))
        readme = os.path.join(path, detected_path, readme_name(repo=r))
        if os.path.exists(readme):
            logging.info(" - {} found at {}".format(r, detected_path))
        else:
            logging.info(" - {} not found".format(r))


def checkout_path(workspace, repo):
    """Get the checkout path for a repo"""
    return workspace + "/" + repo + ".src"


def tree_path(tree, repo):
    """Get the derived path for a repo"""
    if tree == "llvm":
        if repo == "llvm":
            return ""
        if repo == "clang":
            return "tools/clang"
        if repo == "clang-tools-extra":
            return "tools/clang/tools/extra"
        if repo == "debuginfo-tests":
            return "tools/clang/test/debuginfo-tests"
        if repo == "compiler-rt":
            return "projects/compiler-rt"
        if repo == "libcxx":
            return "projects/libcxx"

    elif tree == "lldb":
        if repo == "lldb":
            return ""
        if repo == "llvm":
            return "llvm"
        if repo == "clang":
            return "llvm/tools/clang"

    else:
        logging.error("Unknown tree '{}'".format(tree))
        sys.exit(1)

    logging.error("Unknown repo '{}' in tree '{}".format(repo, tree))
    sys.exit(1)


def tree_srcdir(conf, tree):
    """Get the srcdir for a tree"""
    if tree == "llvm":
        return conf.srcdir()

    if tree == "lldb":
        return conf.lldbsrcdir()

    logging.error("Unknown tree '{}'".format(tree))
    sys.exit(1)


def derived_path(srcdir, tree_path):
    """Get the derived path from a tree path"""
    if tree_path:
        return srcdir + "/" + tree_path
    return srcdir


def should_exclude(base_path, repo_path):
    """Check wither a repo should be excluded in a given rsync"""
    if base_path == repo_path:
        return False
    if not base_path:
        return True
    if repo_path.startswith(base_path + "/"):
        return True
    return False


def rsync(conf, tree, repo, repos):
    """rsync from the checkout to the derived path"""
    cmd = ["rsync", "-auvh", "--delete", "--exclude=.svn/"]
    path = tree_path(tree=tree, repo=repo)
    for x in repos:
        x_path = tree_path(tree=tree, repo=x)
        if should_exclude(path, x_path):
            cmd.append("--exclude=/" + x_path)

    workspace = conf.workspace
    srcdir = tree_srcdir(conf=conf, tree=tree)
    cmd.append(checkout_path(workspace=workspace, repo=repo) + "/")
    cmd.append(derived_path(srcdir=srcdir, tree_path=path))
    run_cmd(working_dir=srcdir, cmd=cmd)


def derive(tree, repos):
    """Build a derived src tree from all the svn repos.

    Try to do this in a way that is pretty fast if the
    derived tree is already there.
    """

    # Check for src dirs.
    for p in repos:
        full_path = checkout_path(workspace=conf.workspace, repo=p)
        if not os.path.exists(full_path):
            logging.error("Cannot find Repo: in " + full_path)
            sys.exit(1)

    # Make sure destinations exist.
    srcdir = tree_srcdir(conf=conf, tree=tree)
    for p in repos:
        full_path = derived_path(srcdir=srcdir, tree_path=tree_path(tree=tree, repo=p))
        if not os.path.exists(full_path):
            os.makedirs(full_path)

    header("Derive Source")
    for repo in repos:
        rsync(conf=conf, tree=tree, repo=repo, repos=repos)
    footer()


def derive_llvm(repos=['llvm', 'clang', 'libcxx', 'clang-tools-extra', \
        'compiler-rt', 'debuginfo-tests']):
    """Build a derived src tree for LLVM"""
    derive(tree='llvm', repos=repos)


def derive_lldb():
    """Build a derived src tree for LLDB"""
    derive(tree='lldb', repos=['lldb', 'llvm', 'clang'])


def create_builddirs():
    paths = [conf.builddir(), conf.installdir()]
    for p in paths:
        if not os.path.exists(p):
            os.makedirs(p)


def fetch_compiler():
    local_name = "host-compiler.tar.gz"
    url = conf.host_compiler_url + "/" + conf.artifact_url
    header("Fetching Compiler")
    print "Fetching:", url,
    urllib.urlretrieve (url, conf.workspace + "/" + local_name)
    print "done."
    print "Decompressing..."
    if not os.path.exists(conf.workspace + "/host-compiler"):
        os.mkdir(conf.workspace + "/host-compiler")
    run_cmd(conf.workspace + "/host-compiler/", ['tar', 'zxf', "../" + local_name])
    os.unlink(local_name)
    footer()


def build_upload_artifact():
    """Create artifact for this build, and upload to server."""
    assert conf.svn_rev != "NONE"
    prop_file = "last_good_build.properties"

    artifact_name = "clang-r{}-t{}-b{}.tar.gz".format(conf.svn_rev,
        conf.build_id, conf.build_number)
    new_url = conf.job_name + "/" + artifact_name

    with open(prop_file, 'w') as prop_fd:
        prop_fd.write("LLVM_REV={}\n".format(conf.svn_rev))
        prop_fd.write("ARTIFACT={}\n".format(new_url))

    tar = ["tar", "zcvf", "../" + artifact_name, "."]

    run_cmd(conf.installdir(), tar)

    upload_cmd = ["scp", artifact_name,
        "buildslave@" + SERVER + ":/Library/WebServer/Documents/artifacts/" +
        conf.job_name + "/"]

    run_cmd(conf.workspace, upload_cmd)

    upload_cmd = ["scp", prop_file,
        "buildslave@" + SERVER + ":/Library/WebServer/Documents/artifacts/" +
        conf.job_name + "/"]

    run_cmd(conf.workspace, upload_cmd)

    ln_cmd = ["ssh", "buildslave@" + SERVER,
        "ln", "-fs", "/Library/WebServer/Documents/artifacts/" +
        conf.job_name + "/" + artifact_name,
        "/Library/WebServer/Documents/artifacts/" +
        conf.job_name + "/latest" ]

    run_cmd(conf.workspace, ln_cmd)


def run_cmd(working_dir, cmd, env=None):
    """Run a command in a working directory, and make sure it returns zero."""
    old_cwd = os.getcwd()
    cmd_to_print = ' '.join([quote_sh_string(x) for x in cmd])
    sys.stdout.write("cd {}\n{}\n".format(working_dir, cmd_to_print))
    sys.stdout.flush()

    start_time = datetime.datetime.now()
    if not os.environ.get('TESTING', False):
        os.chdir(working_dir)
        subprocess.check_call(cmd, env=env)
        os.chdir(old_cwd)
    end_time = datetime.datetime.now()

    logging.info("Command took {} seconds".format(
        (end_time-start_time).seconds))


def run_cmd_errors_okay(working_dir, cmd, env=None):
    """Run a command in a working directory, reporting return value.
    Non-zero exit codes do not generate an exception.
    """
    old_cwd = os.getcwd()
    cmd_to_print = ' '.join([quote_sh_string(x) for x in cmd])
    sys.stdout.write("cd {}\n{}\n".format(working_dir, cmd_to_print))
    sys.stdout.flush()

    start_time = datetime.datetime.now()
    if not os.environ.get('TESTING', False):
        try:
            os.chdir(working_dir)
            result = subprocess.call(cmd, env=env)
        finally:
            os.chdir(old_cwd)
    end_time = datetime.datetime.now()

    logging.info("Command took {} seconds: return code {}".format(
        (end_time-start_time).seconds, result))

KNOWN_TARGETS = ['all', 'build', 'test', 'testlong']
KNOWN_BUILDS = ['clang', 'cmake', 'lldb', 'fetch', 'artifact',
                'derive', 'derive-llvm+clang', 'derive-lldb', 'derive-llvm',
                'static-analyzer-benchmarks']


def parse_args():
    """Get the command line arguments, and make sure they are correct."""

    parser = argparse.ArgumentParser(
        description='Build and test compilers and other things.')

    parser.add_argument("build_type",
                        help="The kind of build to trigger.",
                        choices=KNOWN_BUILDS)

    parser.add_argument("build_target",
                        nargs='?',
                        help="The targets to call (build, check, etc).",
                        choices=KNOWN_TARGETS)

    parser.add_argument('--assertions', dest='assertions', action='store_true')
    parser.add_argument('--lto', dest='lto', action='store_true')
    parser.add_argument('--debug', dest='debug', action='store_true')
    parser.add_argument('--cmake-type', dest='cmake_build_type',
                        help="Override cmake type Release, Debug, "
                        "RelWithDebInfo and MinSizeRel")
    parser.add_argument('--cmake-flag', dest='cmake_flags',
                        action='append', default=[],
                        help='Set an arbitrary cmake flag')
    parser.add_argument('--compiler-flag', dest='compiler_flags',
                        action='append', default=[],
                        help='Set an arbitrary compiler flag')
    args = parser.parse_args()
    return args


def main():
    """Run a build based on command line args and ENV."""
    global conf
    args = parse_args()
    conf = Configuration(args)

    create_builddirs()
    try:
        if args.build_type == 'clang':
            clang_builder(args.build_target)
        elif args.build_type == 'lldb':
            lldb_builder()
        elif args.build_type == 'cmake':
            cmake_builder(args.build_target)
        elif args.build_type == 'derive':
            derive_llvm()
        elif args.build_type == 'derive-llvm+clang':
            derive_llvm(['llvm', 'clang'])
        elif args.build_type == 'derive-llvm':
            derive_llvm(['llvm'])
        elif args.build_type == 'derive-lldb':
            derive_lldb()
        elif args.build_type == 'fetch':
            fetch_compiler()
        elif args.build_type == 'artifact':
            build_upload_artifact()
        elif args.build_type == 'static-analyzer-benchmarks':
            static_analyzer_benchmarks_builder()
    except subprocess.CalledProcessError as exct:
        print "Command failed", exct.message
        print "Command:", exct.cmd
        sys.exit(1)


if __name__ == '__main__':
    main()