path: root/lnt/tests/compile.py

"""Single file compile-time performance testing"""
from __future__ import print_function
import errno
import hashlib
import json
import os
import platform
import pprint
import re
import shlex
import shutil
import subprocess
import sys
import logging
from datetime import datetime
import collections

import click

import lnt.testing
import lnt.testing.util.compilers
from lnt.testing.util import commands, machineinfo
from lnt.testing.util.commands import fatal, resolve_command_path
from lnt.testing.util.misc import timestamp
from lnt.tests import builtintest
from lnt.util import stats
from lnt.util import logger
from lnt.lnttool.common import submit_options


# For each test, compile with all these combinations of flags.
DEFAULT_FLAGS_TO_TEST = [('-O0',),
                         ('-O0', '-g',),
                         ('-Os',),
                         ('-Os', '-g'),
                         ('-O3',),
                         ('-O3', '-g'),
                         ('-Oz',),
                         ('-Oz', '-g')]

opts = None


def args_to_quoted_string(args):
    def quote_arg(arg):
        if "'" in arg or '(' in arg or ')' in arg:
            return '"%s"' % arg
        elif '"' in arg or ' ' in arg:
            return "'%s'" % arg
        return arg
    return ' '.join([quote_arg(a)
                     for a in args])


def runN(args, N, cwd, preprocess_cmd=None, env=None, sample_mem=False,
         ignore_stderr=False, stdout=None, stderr=None):
    """Interface to runN.

    FIXME: Figure out a better way to deal with need to run as root. Maybe farm
    memory sampling process out into something we can setuid? Eek.
    """
    g_log.info("preprocess_cmd at top of runN: %s:", preprocess_cmd)
    cmd = [opts.runn, '-a']
    if sample_mem:
        cmd = ['sudo'] + cmd + ['-m']
    if preprocess_cmd is not None:
        cmd.extend(('-p', preprocess_cmd))
    if stdout is not None:
        cmd.extend(('--stdout', stdout))
    if stderr is not None:
        cmd.extend(('--stderr', stderr))
    cmd.extend(('--min-sample-time', repr(opts.min_sample_time)))
    cmd.extend(('--max-num-samples', '100'))
    cmd.append(str(int(N)))
    cmd.extend(args)

    if opts.verbose:
        g_log.info("running: %s" % " ".join("'%s'" % arg for arg in cmd))
    p = subprocess.Popen(args=cmd,
                         stdin=None,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE,
                         env=env,
                         cwd=cwd)
    runn_stdout, runn_stderr = p.communicate()
    res = p.returncode

    # If the runN command failed, or it had stderr when we didn't expect it,
    # fail immediately and don't try to parse the output.
    if res != 0:
        g_log.error("runN command failed with stderr:\n--\n%s\n--" % runn_stderr.strip())
        return None
    elif not ignore_stderr and runn_stderr.strip():
        g_log.error("command had unexpected output on stderr:\n--\n%s\n--" % (
                    runn_stderr.strip(),))
        return None

    # Otherwise, parse the timing data from runN.
    try:
        return eval(runn_stdout)
    except Exception:
        fatal("failed to parse output: %s\n" % runn_stdout)


# Test functions.
def get_input_path(opts, *names):
    return os.path.join(opts.test_suite_externals, opts.test_subdir,
                        *names)


def get_output_path(*names):
    return os.path.join(g_output_dir, *names)


def get_runN_test_data(name, variables, cmd, ignore_stderr=False,
                       sample_mem=False, only_mem=False,
                       stdout=None, stderr=None, preprocess_cmd=None,
                       env=None):
    if only_mem and not sample_mem:
        raise Exception("only_mem doesn't make sense without sample_mem")

    data = runN(cmd, variables.get('run_count'), cwd='/tmp',
                ignore_stderr=ignore_stderr, sample_mem=sample_mem,
                stdout=stdout, stderr=stderr, preprocess_cmd=preprocess_cmd,
                env=env)
    if data is not None:
        if data.get('version') != 0:
            raise ValueError('unknown runN data format')
        data_samples = data.get('samples')
    else:
        # Print stdout/stderr log if available
        if stdout is not None and os.path.exists(stdout):
            g_log.info("%s:\n--\n%s\n--" % (stdout, open(stdout).read()))
        if stderr is not None and os.path.exists(stderr):
            g_log.info("%s:\n--\n%s\n--" % (stderr, open(stderr).read()))
    keys = []
    if not only_mem:
        keys.extend([('user', 1,), ('sys', 2,), ('wall', 3,)])
    if sample_mem:
        keys.append(('mem', 4))
    for key, idx in keys:
        tname = '%s.%s' % (name, key)
        success = False
        samples = []
        if data is not None:
            success = True
            samples = [sample[idx] for sample in data_samples]
        yield (success, tname, samples)


# FIXME: Encode dependency on output automatically, for simpler test execution.
def test_cc_command(base_name, run_info, variables, input, output, flags,
                    extra_flags, has_output=True, ignore_stderr=False):
    name = '%s/(%s)' % (base_name, ' '.join(flags),)
    input = get_input_path(opts, input)
    output = get_output_path(output)

    cmd = [variables.get('cc')]
    cmd.extend(extra_flags)
    cmd.append(input)
    cmd.extend(flags)

    # Inhibit all warnings, we don't want to count the time to generate them
    # against newer compilers which have added (presumably good) warnings.
    cmd.append('-w')

    # Do a memory profiling run, if requested.
    #
    # FIXME: Doing this as a separate step seems silly. We shouldn't do any
    # extra run just to get the memory statistics.
    if opts.memory_profiling:
        # Find the cc1 command, which we use to do memory profiling. To do this
        # we execute the compiler with '-###' to figure out what it wants to
        # do.
        cc_output = commands.capture(cmd + ['-o', '/dev/null', '-###'],
                                     include_stderr=True).strip()
        cc_commands = []
        for ln in cc_output.split('\n'):
            # Filter out known garbage.
            is_garbage = bool(ln == 'Using built-in specs.' or
                              ln.startswith('Configured with:') or
                              ln.startswith('Target:') or
                              ln.startswith('Thread model:') or
                              ln.startswith('InstalledDir:') or
                              ' version ' in ln)
            if is_garbage:
                    continue
            cc_commands.append(ln)

        if len(cc_commands) != 1:
            fatal('unable to determine cc1 command: %r' % cc_output)

        cc1_cmd = shlex.split(cc_commands[0])
        for res in get_runN_test_data(name, variables, cc1_cmd,
                                      ignore_stderr=ignore_stderr,
                                      sample_mem=True, only_mem=True):
            yield res

    commands.rm_f(output)
    for res in get_runN_test_data(name, variables, cmd + ['-o', output],
                                  ignore_stderr=ignore_stderr):
        yield res

    # If the command has output, track its size.
    if has_output:
        tname = '%s.size' % (name,)
        success = False
        samples = []
        try:
            stat = os.stat(output)
            success = True

            # For now, the way the software is set up things are going to get
            # confused if we don't report the same number of samples as
            # reported for other variables. So we just report the size N times.
            #
            # FIXME: We should resolve this, eventually.
            for i in range(variables.get('run_count')):
                samples.append(stat.st_size)
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise
        yield (success, tname, samples)


Stage = collections.namedtuple("Stage", ['flags', 'has_output'])

PCH_GEN = "pch-gen"
DRIVER = "driver"
INIT = "init"
SYNTAX = "syntax"
IRGEN_ONLY = "irgen_only"
IRGEN = "irgen"
CODEGEN = "codegen"
ASSEMBLY = "assembly"

STAGE_TO_FLAG_MAP = {
    PCH_GEN: Stage(flags=['-x', 'objective-c-header'], has_output=True),
    DRIVER: Stage(flags=['-###', '-fsyntax-only'], has_output=False),
    INIT: Stage(flags=['-fsyntax-only', '-Xclang', '-init-only'],
                has_output=False),
    SYNTAX: Stage(flags=['-fsyntax-only'], has_output=False),
    IRGEN_ONLY: Stage(flags=['-emit-llvm', '-c', '-Xclang', '-emit-llvm-only'],
                      has_output=False),
    IRGEN: Stage(flags=['-emit-llvm', '-c'], has_output=True),
    CODEGEN: Stage(flags=['-c', '-Xclang', '-emit-codegen-only'],
                   has_output=False),
    # Object would be better name. Keep for backwards compat.
    ASSEMBLY: Stage(flags=['-c'], has_output=True),
}


def test_compile(name, run_info, variables, input, output, pch_input,
                 flags, stage, extra_flags=[]):
    extra_flags = list(extra_flags)

    cc_name = variables.get('cc_name')
    is_llvm = not (cc_name == 'gcc')
    is_clang = not (cc_name in ('gcc', 'llvm-gcc'))

    # Ignore irgen stages for non-LLVM compilers.
    if not is_llvm and stage in (IRGEN, IRGEN_ONLY):
        return ()

    # Ignore 'init', 'irgen_only', and 'codegen' stages for non-Clang.
    if not is_clang and stage in (INIT, IRGEN_ONLY, CODEGEN):
        return ()

    # Force gnu99 mode for all compilers.
    if not is_clang:
        extra_flags.append('-std=gnu99')

    stage_flags, has_output = STAGE_TO_FLAG_MAP[stage]

    # Ignore stderr output (instead of failing) in 'driver' stage, -### output
    # goes to stderr by default.
    ignore_stderr = stage == DRIVER

    extra_flags.extend(stage_flags)
    if pch_input is not None:
        assert pch_input.endswith('.gch')
        extra_flags.extend(['-include', get_output_path(pch_input[:-4])])

    extra_flags.extend(['-I', os.path.dirname(get_input_path(opts, input))])

    return test_cc_command(name, run_info, variables, input, output, flags,
                           extra_flags, has_output, ignore_stderr)


def test_build(base_name, run_info, variables, project, build_config, num_jobs,
               codesize_util=None):
    name = '%s(config=%r,j=%d)' % (base_name, build_config, num_jobs)

    # Check if we need to expand the archive into the sandbox.
    archive_path = get_input_path(opts, project['archive'])
    with open(archive_path) as f:
        archive_hash = hashlib.md5(f.read() + str(project)).hexdigest()

    # Compute the path to unpack to.
    source_path = get_output_path("..", "Sources", project['name'])

    # Load the hash of the last unpack, in case the archive has been updated.
    last_unpack_hash_path = os.path.join(source_path, "last_unpack_hash.txt")
    if os.path.exists(last_unpack_hash_path):
        with open(last_unpack_hash_path) as f:
            last_unpack_hash = f.read()
    else:
        last_unpack_hash = None

    # Unpack if necessary.
    if last_unpack_hash == archive_hash:
        g_log.info('reusing sources %r (already unpacked)' % name)
    else:
        # Remove any existing content, if necessary.
        try:
            shutil.rmtree(source_path)
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise

        # Extract the zip file.
        #
        # We shell out to unzip here because zipfile's extractall does not
        # appear to preserve permissions properly.
        commands.mkdir_p(source_path)
        g_log.info('extracting sources for %r' % name)

        if archive_path.endswith(".tar.gz") or \
           archive_path.endswith(".tar.bz2") or \
           archive_path.endswith(".tar.lzma"):
            p = subprocess.Popen(args=['tar', '-xf', archive_path],
                                 stdin=None,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE,
                                 cwd=source_path)
        else:
            p = subprocess.Popen(args=['unzip', '-q', archive_path],
                                 stdin=None,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE,
                                 cwd=source_path)
        stdout, stderr = p.communicate()
        if p.wait() != 0:
            fatal(("unable to extract archive %r at %r\n"
                   "-- stdout --\n%s\n"
                   "-- stderr --\n%s\n") % (archive_path, source_path,
                                            stdout, stderr))

        # Apply the patch file, if necessary.
        patch_files = project.get('patch_files', [])
        for patch_file in patch_files:
            g_log.info('applying patch file %r for %r' % (patch_file, name))
            patch_file_path = get_input_path(opts, patch_file)
            p = subprocess.Popen(args=['patch', '-i', patch_file_path,
                                       '-p', '1'],
                                 stdin=None,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE,
                                 cwd=source_path)
            stdout, stderr = p.communicate()
            if p.wait() != 0:
                fatal(("unable to apply patch file %r in %r\n"
                       "-- stdout --\n%s\n"
                       "-- stderr --\n%s\n") % (patch_file_path, source_path,
                                                stdout, stderr))

        # Write the hash tag.
        with open(last_unpack_hash_path, "w") as f:
            f.write(archive_hash)

    # Create an env dict in case the user wants to use it.
    env = dict(os.environ)

    # Form the test build command.
    build_info = project['build_info']

    # Add arguments to ensure output files go into our build directory.
    dir_name = '%s_%s_j%d' % (base_name, build_config, num_jobs)
    output_base = get_output_path(dir_name)
    build_base = os.path.join(output_base, 'build', build_config)

    # Create the build base directory and by extension output base directory.
    commands.mkdir_p(build_base)

    cmd = []
    preprocess_cmd = None

    if build_info['style'].startswith('xcode-'):
        file_path = os.path.join(source_path, build_info['file'])
        cmd.extend(['xcodebuild'])

        # Add the arguments to select the build target.
        if build_info['style'] == 'xcode-project':
            cmd.extend(('-target', build_info['target'],
                        '-project', file_path))
        elif build_info['style'] == 'xcode-workspace':
            cmd.extend(('-scheme', build_info['scheme'],
                        '-workspace', file_path))
            cmd.extend(('-derivedDataPath', build_base))
        else:
            fatal("unknown build style in project: %r" % project)

        # Add the build configuration selection.
        cmd.extend(('-configuration', build_config))

        cmd.append('OBJROOT=%s' % os.path.join(build_base, 'obj'))
        cmd.append('SYMROOT=%s' % os.path.join(build_base, 'sym'))
        cmd.append('DSTROOT=%s' % os.path.join(build_base, 'dst'))
        cmd.append('SHARED_PRECOMPS_DIR=%s' % os.path.join(build_base, 'pch'))

        # Add arguments to force the appropriate compiler.
        cmd.append('CC=%s' % (opts.cc,))
        cmd.append('CPLUSPLUS=%s' % (opts.cxx,))

        # We need to force this variable here because Xcode has some completely
        # broken logic for deriving this variable from the compiler
        # name. <rdar://problem/7989147>
        cmd.append('LD=%s' % (opts.ld,))
        cmd.append('LDPLUSPLUS=%s' % (opts.ldxx,))

        # Force off the static analyzer, in case it was enabled in any projects
        # (we don't want to obscure what we are trying to time).
        cmd.append('RUN_CLANG_STATIC_ANALYZER=NO')

        # Inhibit all warnings, we don't want to count the time to generate
        # them against newer compilers which have added (presumably good)
        # warnings.
        cmd.append('GCC_WARN_INHIBIT_ALL_WARNINGS=YES')

        # Add additional arguments to force the build scenario we want.
        cmd.extend(('-jobs', str(num_jobs)))

        # If the user specifies any additional options to be included on the
        # command line, append them here.
        cmd.extend(build_info.get('extra_args', []))

        # If the user specifies any extra environment variables, put
        # them in our env dictionary.
        env_format = {
            'build_base': build_base
        }
        extra_env = build_info.get('extra_env', {})
        for k in extra_env:
            extra_env[k] = extra_env[k] % env_format
        env.update(extra_env)

        # Create preprocess cmd
        preprocess_cmd = 'rm -rf "%s"' % (build_base,)

    elif build_info['style'] == 'make':
        # Get the subdirectory in Source where our sources exist.
        src_dir = os.path.dirname(os.path.join(source_path,
                                               build_info['file']))
        # Grab our config from build_info. This is config is currently only
        # used in the make build style since Xcode, the only other build style
        # as of today, handles changing configuration through the configuration
        # type variables.  Make does not do this so we have to use more brute
        # force to get it right.
        config = build_info.get('config', {}).get(build_config, {})

        # Copy our source directory over to build_base.
        # We do this since we assume that we are processing a make project
        # which has already been configured and so that we do not need to worry
        # about make install or anything like that. We can just build the
        # project and use the user supplied path to its location in the build
        # directory.
        copied_src_dir = os.path.join(build_base, os.path.basename(dir_name))
        shutil.copytree(src_dir, copied_src_dir)

        # Create our make command.
        cmd.extend(['make', '-C', copied_src_dir, build_info['target'], "-j",
                    str(num_jobs)])

        # If the user specifies any additional options to be included on the
        # command line, append them here.
        cmd.extend(config.get('extra_args', []))

        # If the user specifies any extra environment variables, put
        # them in our env dictionary.

        # We create a dictionary for build_base so that users can use
        # it optionally in an environment variable via the python
        # format %(build_base)s.
        env_format = {
            'build_base': build_base
        }

        extra_env = config.get('extra_env', {})
        for k in extra_env:
            extra_env[k] = extra_env[k] % env_format
        env.update(extra_env)

        # Set build base to copied_src_dir so that if codesize_util
        # is not None, we pass it the correct path.
        build_base = copied_src_dir
        preprocess_cmd = 'rm -rf "%s"/build' % (build_base,)
        g_log.info('preprocess_cmd: %s' % preprocess_cmd)

    else:
        fatal("unknown build style in project: %r" % project)

    # Collect the samples.
    g_log.info('executing full build: %s' % args_to_quoted_string(cmd))
    stdout_path = os.path.join(output_base, "stdout.log")
    stderr_path = os.path.join(output_base, "stderr.log")

    for res in get_runN_test_data(name, variables, cmd,
                                  stdout=stdout_path, stderr=stderr_path,
                                  preprocess_cmd=preprocess_cmd, env=env):
        yield res

    # If we have a binary path, get the text size of our result.
    binary_path = build_info.get('binary_path', None)
    if binary_path is not None and codesize_util is not None:
        tname = "%s.size" % (name,)
        success = False
        samples = []

        try:
            # We use a dictionary here for our formatted processing of
            # binary_path so that if the user needs our build config he can get
            # it via %(build_config)s in his string and if he does not, an
            # error is not thrown.
            format_args = {"build_config": build_config}
            cmd = codesize_util + [os.path.join(build_base,
                                                binary_path % format_args)]
            if opts.verbose:
                g_log.info('running: %s' %
                           " ".join("'%s'" % arg for arg in cmd))
            result = subprocess.check_output(cmd).strip()
            if result != "fail":
                bytes = int(result)
                success = True

                # For now, the way the software is set up things are going to
                # get confused if we don't report the same number of samples
                # as reported for other variables. So we just report the size
                # N times.
                #
                # FIXME: We should resolve this, eventually.
                for i in range(variables.get('run_count')):
                    samples.append(bytes)
            else:
                g_log.warning('Codesize failed.')

        except OSError as e:
            if e.errno != errno.ENOENT:
                raise
            else:
                g_log.warning('Codesize failed with ENOENT.')
        yield (success, tname, samples)

    # Check that the file sizes of the output log files "make sense", and warn
    # if they do not. That might indicate some kind of non-determinism in the
    # test command, which makes timing less useful.
    stdout_sizes = []
    stderr_sizes = []
    run_count = variables['run_count']
    for i in range(run_count):
        iter_stdout_path = '%s.%d' % (stdout_path, i)
        iter_stderr_path = '%s.%d' % (stderr_path, i)
        if os.path.exists(iter_stdout_path):
            stdout_sizes.append(os.stat(iter_stdout_path).st_size)
        else:
            stdout_sizes.append(None)
        if os.path.exists(iter_stderr_path):
            stderr_sizes.append(os.stat(iter_stderr_path).st_size)
        else:
            stderr_sizes.append(None)

    if len(set(stdout_sizes)) != 1:
        g_log.warning(('test command had stdout files with '
                       'different sizes: %r') % stdout_sizes)
    if len(set(stderr_sizes)) != 1:
        g_log.warning(('test command had stderr files with '
                       'different sizes: %r') % stderr_sizes)

    # Unless cleanup is disabled, rerun the preprocessing command.
    if not opts.save_temps and preprocess_cmd:
        g_log.info('cleaning up temporary results')
        if os.system(preprocess_cmd) != 0:
            g_log.warning("cleanup command returned a non-zero exit status")


def curry(fn, **kw_args):
    return lambda *args: fn(*args, **kw_args)


def get_single_file_tests(flags_to_test, test_suite_externals,
                          subdir):
    # Load the project description file from the externals.
    path = os.path.join(test_suite_externals, subdir,
                        "project_list.json")
    with open(path) as f:
        config = json.load(f).get("single-file", {})

        if len(config) == 0:
            g_log.warning("config file %s has no data." % path)

        all_pch = config.get("pch", [])
        all_inputs = config.get("tests", [])

    stages_to_test = [DRIVER, INIT, SYNTAX, IRGEN_ONLY, IRGEN,
                      CODEGEN, ASSEMBLY]
    base_path = os.path.join(test_suite_externals, subdir, 'single-file')

    if not os.access(base_path, os.F_OK | os.R_OK):
        g_log.warning('single-file directory does not exist. Dir: %s'
                      % base_path)
    else:
        # I did not want to handle the control flow in this manner, but due to
        # the nature of python generators I can not just return in the previous
        # warning case.
        for f in flags_to_test:
            # FIXME: Note that the order matters here, because we need to make
            # sure to generate the right PCH file before we try to use it.
            # Ideally the testing infrastructure would just handle this.
            for pch in all_pch:
                path, name, output = pch['path'], pch['name'], pch['output']

                yield (os.path.join('pch-gen', name),
                       curry(test_compile,
                             input=os.path.join(base_path, path),
                             output=output, pch_input=None,
                             flags=f, stage='pch-gen'))

            for input in all_inputs:
                path, pch_input = input['path'], input.get('pch_input', None)
                extra_flags = input['extra_flags']

                name = path
                output = os.path.splitext(os.path.basename(path))[0] + '.o'
                for stage in stages_to_test:
                    yield ('compile/%s/%s' % (name, stage),
                           curry(test_compile,
                                 input=os.path.join(base_path, path),
                                 output=output, pch_input=pch_input, flags=f,
                                 stage=stage, extra_flags=extra_flags))


def get_full_build_tests(jobs_to_test, configs_to_test,
                         test_suite_externals, test_suite_externals_subdir):
    # Load the project description file from the externals.
    path = os.path.join(test_suite_externals, test_suite_externals_subdir,
                        "project_list.json")

    g_log.info("Loading config file: %s" % path)
    with open(path) as f:
        data = json.load(f)

    codesize_util = data.get('codesize_util', None)

    for jobs in jobs_to_test:
        for project in data['projects']:
            for config in configs_to_test:
                # Check the style.
                yield ('build/%s' % (project['name'],),
                       curry(test_build, project=project, build_config=config,
                             num_jobs=jobs, codesize_util=codesize_util))


def get_tests(test_suite_externals, test_suite_externals_subdir, flags_to_test,
              jobs_to_test, configs_to_test):
    for item in get_single_file_tests(flags_to_test, test_suite_externals,
                                      test_suite_externals_subdir):
        yield item

    for item in get_full_build_tests(jobs_to_test, configs_to_test,
                                     test_suite_externals,
                                     test_suite_externals_subdir):
        yield item


g_output_dir = None
g_log = None
usage_info = """
Script for testing compile time performance.

This tests:
 - PCH Generation for Cocoa.h
   o File Size
   o Memory Usage
   o Time
 - Objective-C Compile Time, with PCH
   o File Sizes
   o Memory Usage
   o Time
 - C Compile Time, without PCH
   o File Sizes
   o Memory Usage
   o Time
 - Full Build Times
   o Total Build Time (using xcodebuild)

TODO:
 - Objective-C Compile Time, with PCH
   o PCH Utilization

FIXME: One major hole here is that we aren't testing one situation which does
sometimes show up with PCH, where we have a PCH file + a second significant
body of code (e.g., a large user framework, or a poorly PCHified project). In
practice, this can be a significant hole because PCH has a substantial impact
on how lookup, for example, is done.

We run each of the tests above in a number of dimensions:
 - O0
 - O0 -g
 - Os

We run each of the compile time tests in various stages:
 - ### (driver time)
 - init (driver + compiler init)
 - fsyntax-only (lex/parse/sema time)
 - emit-llvm-only (IRgen time)
 - emit-llvm (.bc output time and size, mostly to track output file size)
 - emit-codegen-only (codegen time, without assembler)
 - c (assembly time and size)
"""


class CompileTest(builtintest.BuiltinTest):
    def run_test(self, opts):

        # Resolve the cc_under_test path.
        opts.cc = resolve_command_path(opts.cc)

        if not lnt.testing.util.compilers.is_valid(opts.cc):
            self._fatal('--cc does not point to a valid executable.')

        # Attempt to infer the cxx compiler if not given.
        if opts.cc and opts.cxx is None:
            opts.cxx = lnt.testing.util.compilers.infer_cxx_compiler(opts.cc)
            if opts.cxx is not None:
                logger.info("inferred C++ compiler under test as: %r" %
                            (opts.cxx,))

        if opts.cxx is None:
            self._fatal('--cxx is required (and could not be inferred)')

        # Force the CC and CXX variables to be absolute paths.
        cc_abs = os.path.abspath(commands.which(opts.cc))
        cxx_abs = os.path.abspath(commands.which(opts.cxx))

        if not os.path.exists(cc_abs):
            self._fatal("unable to determine absolute path for --cc: %r" % (
                opts.cc,))
        if not os.path.exists(cxx_abs):
            self._fatal("unable to determine absolute path for --cc: %r" % (
                opts.cc,))
        opts.cc = cc_abs
        opts.cxx = cxx_abs

        # If no ld was set, set ld to opts.cc
        if opts.ld is None:
            opts.ld = opts.cc
        # If no ldxx was set, set ldxx to opts.cxx
        if opts.ldxx is None:
            opts.ldxx = opts.cxx

        # Set up the sandbox.
        global g_output_dir
        if not os.path.exists(opts.sandbox_path):
            print("%s: creating sandbox: %r" % (
                timestamp(), opts.sandbox_path), file=sys.stderr)
            os.mkdir(opts.sandbox_path)
        if opts.timestamp_build:
            fmt_timestamp = timestamp().replace(' ', '_').replace(':', '-')
            report_name = "test-%s" % (fmt_timestamp)
        else:
            report_name = "build"
        g_output_dir = os.path.join(os.path.abspath(opts.sandbox_path),
                                    report_name)

        try:
            os.mkdir(g_output_dir)
        except OSError as e:
            if e.errno == errno.EEXIST:
                self._fatal("sandbox output directory %r already exists!" % (
                    g_output_dir,))
            else:
                raise

        # Setup log file
        global g_log

        def setup_log(output_dir):
            def stderr_log_handler():
                h = logging.StreamHandler()
                f = logging.Formatter(
                    "%(asctime)-7s: %(levelname)s: %(message)s",
                    "%Y-%m-%d %H:%M:%S")
                h.setFormatter(f)
                return h

            def file_log_handler(path):
                h = logging.FileHandler(path, mode='w')
                f = logging.Formatter(
                    "%(asctime)-7s: %(levelname)s: %(message)s",
                    "%Y-%m-%d %H:%M:%S")
                h.setFormatter(f)
                return h
            log = logging.Logger('compile_test')
            log.setLevel(logging.INFO)
            log.addHandler(file_log_handler(os.path.join(output_dir,
                                                         'test.log')))
            log.addHandler(stderr_log_handler())
            return log
        g_log = setup_log(g_output_dir)

        # Collect machine and run information.
        machine_info, run_info = machineinfo.get_machine_information(
            opts.use_machdep_info)

        # FIXME: Include information on test source versions.
        #
        # FIXME: Get more machine information? Cocoa.h hash, for example.

        for name, cmd in (('sys_cc_version', ('/usr/bin/gcc', '-v')),
                          ('sys_as_version',
                           ('/usr/bin/as', '-v', '/dev/null')),
                          ('sys_ld_version', ('/usr/bin/ld', '-v')),
                          ('sys_xcodebuild', ('xcodebuild', '-version'))):
            run_info[name] = commands.capture(cmd, include_stderr=True).strip()

        # Set command line machine and run information.
        for info, params in ((machine_info, opts.machine_parameters),
                             (run_info, opts.run_parameters)):
            for entry in params:
                if '=' not in entry:
                    name, value = entry, ''
                else:
                    name, value = entry.split('=', 1)
                info[name] = value

        # Set user variables.
        variables = {}
        variables['cc'] = opts.cc
        variables['run_count'] = opts.run_count

        # Get compiler info.
        cc_info = lnt.testing.util.compilers.get_cc_info(variables['cc'])
        variables.update(cc_info)

        # Set the run order from the user, if given.
        if opts.run_order is not None:
            variables['run_order'] = opts.run_order
        else:
            # Otherwise, use the inferred run order.
            variables['run_order'] = cc_info['inferred_run_order']
            logger.info("inferred run order to be: %r" %
                        (variables['run_order'],))

        if opts.verbose:
            format = pprint.pformat(variables)
            msg = '\n\t'.join(['using variables:'] + format.splitlines())
            logger.info(msg)

            format = pprint.pformat(machine_info)
            msg = '\n\t'.join(['using machine info:'] + format.splitlines())
            logger.info(msg)

            format = pprint.pformat(run_info)
            msg = '\n\t'.join(['using run info:'] + format.splitlines())
            logger.info(msg)

        # Compute the set of flags to test.
        if not opts.flags_to_test:
            flags_to_test = DEFAULT_FLAGS_TO_TEST
        else:
            flags_to_test = [string.split(' ')
                             for string in opts.flags_to_test]

        # Compute the set of job counts to use in full build tests.
        if not opts.jobs_to_test:
            jobs_to_test = [1, 2, 4, 8]
        else:
            jobs_to_test = opts.jobs_to_test

        # Compute the build configurations to test.
        if not opts.configs_to_test:
            configs_to_test = ['Debug', 'Release']
        else:
            configs_to_test = opts.configs_to_test

        # Compute the list of all tests.
        all_tests = list(get_tests(opts.test_suite_externals, opts.test_subdir,
                                   flags_to_test, jobs_to_test,
                                   configs_to_test))

        # Show the tests, if requested.
        if opts.show_tests:
            print('Available Tests', file=sys.stderr)
            for name in sorted(set(name for name, _ in all_tests)):
                print('  %s' % (name, ), file=sys.stderr)
            print()
            raise SystemExit

        # Find the tests to run.
        if not opts.tests and not opts.test_filters:
            tests_to_run = list(all_tests)
        else:
            all_test_names = set(test[0] for test in all_tests)

            # Validate the test names.
            requested_tests = set(opts.tests)
            missing_tests = requested_tests - all_test_names
            if missing_tests:
                self._fatal(("invalid test names %s, use --show-tests to "
                             "see available tests") %
                            (", ".join(map(repr, missing_tests)), ))

            # Validate the test filters.
            test_filters = [re.compile(pattern)
                            for pattern in opts.test_filters]

            # Form the list of tests.
            tests_to_run = [test
                            for test in all_tests
                            if (test[0] in requested_tests or
                                [True
                                 for filter in test_filters
                                 if filter.search(test[0])])]
        if not tests_to_run:
            self._fatal("no tests requested "
                        "(invalid --test or --test-filter options)!")

        # Ensure output directory is available.
        if not os.path.exists(g_output_dir):
            os.mkdir(g_output_dir)

        # Execute the run.
        run_info.update(variables)
        run_info['tag'] = tag = 'compile'

        testsamples = []
        start_time = datetime.utcnow()
        g_log.info('run started')
        g_log.info('using CC: %r' % opts.cc)
        g_log.info('using CXX: %r' % opts.cxx)
        no_errors = True
        for basename, test_fn in tests_to_run:
            for success, name, samples in test_fn(basename, run_info,
                                                  variables):
                g_log.info('collected samples: %r' % name)
                num_samples = len(samples)
                if num_samples:
                    samples_median = '%.4f' % (stats.median(samples),)
                    samples_mad = '%.4f' % (
                        stats.median_absolute_deviation(samples),)
                else:
                    samples_median = samples_mad = 'N/A'
                g_log.info('N=%d, median=%s, MAD=%s' % (
                    num_samples, samples_median, samples_mad))
                test_name = '%s.%s' % (tag, name)
                if not success:
                    testsamples.append(lnt.testing.TestSamples(
                        test_name + '.status', [lnt.testing.FAIL]))
                    no_errors = False
                if samples:
                    testsamples.append(lnt.testing.TestSamples(
                        test_name, samples))
        run_info['no_errors'] = no_errors
        end_time = datetime.utcnow()

        g_log.info('run complete')

        # Package up the report.
        machine = lnt.testing.Machine(opts.machine_name, machine_info)
        run = lnt.testing.Run(start_time, end_time, info=run_info)

        # Write out the report.
        lnt_report_path = os.path.join(g_output_dir, 'report.json')
        report = lnt.testing.Report(machine, run, testsamples)

        # Save report to disk for submission.
        self.print_report(report, lnt_report_path)

        # Then, also print to screen if requested.
        if opts.output is not None:
            self.print_report(report, opts.output)

        server_report = self.submit(lnt_report_path, opts, ts_name='compile')

        return server_report


# FIXME: an equivalent to argparse's add_argument_group is not implemented
#        on click. Need to review it when such functionality is available.
#        https://github.com/pallets/click/issues/373
@click.command("compile", help=usage_info, short_help=__doc__)
@click.argument("label", default=platform.uname()[1], required=False,
                type=click.UNPROCESSED)
@click.option("-s", "--sandbox", "sandbox_path", required=True,
              help="Parent directory to build and run tests in",
              type=click.UNPROCESSED, default=None, metavar="PATH")
#  Test Options
@click.option("--no-timestamp", "timestamp_build",
              help="Don't timestamp build directory (for testing)",
              flag_value=False, default=True)
@click.option("--cc", "cc", type=click.UNPROCESSED, required=True,
              help="Path to the compiler under test")
@click.option("--cxx", "cxx",
              help="Path to the C++ compiler to test",
              type=click.UNPROCESSED, default=None)
@click.option("--ld", "ld",
              help="Path to the c linker to use. (Xcode Distinction)",
              type=click.UNPROCESSED, default=None)
@click.option("--ldxx", "ldxx",
              help="Path to the cxx linker to use. (Xcode Distinction)",
              type=click.UNPROCESSED, default=None)
@click.option("--runn", "runn",
              help="Path to runN tool.",
              type=click.UNPROCESSED, default="runN")
@click.option("--test-externals", "test_suite_externals", required=True,
              help="Path to the LLVM test-suite externals",
              type=click.UNPROCESSED, default=None, metavar="PATH")
@click.option("--machine-param", "machine_parameters",
              metavar="NAME=VAL",
              help="Add 'NAME' = 'VAL' to the machine parameters",
              type=click.UNPROCESSED, multiple=True, default=[])
@click.option("--run-param", "run_parameters",
              metavar="NAME=VAL",
              help="Add 'NAME' = 'VAL' to the run parameters",
              type=click.UNPROCESSED, multiple=True, default=[])
@click.option("--run-order", "run_order", metavar="STR",
              help="String to use to identify and order this run",
              type=click.UNPROCESSED, default=None)
@click.option("--test-subdir", "test_subdir",
              help="Subdirectory of test external dir to look for "
                   "tests in.",
              type=click.UNPROCESSED, default="lnt-compile-suite-src")
#  Test Selection
@click.option("--no-memory-profiling", "memory_profiling",
              help="Disable memory profiling",
              flag_value=False, default=True)
@click.option("--multisample", "run_count", metavar="N",
              help="Accumulate test data from multiple runs",
              type=int, default=3)
@click.option("--min-sample-time", "min_sample_time",
              help="Ensure all tests run for at least N seconds",
              metavar="N", type=float, default=.5)
@click.option("--save-temps", "save_temps",
              help="Save temporary build output files", is_flag=True)
@click.option("--show-tests", "show_tests",
              help="Only list the availables tests that will be run",
              is_flag=True)
@click.option("--test", "tests", metavar="NAME",
              help="Individual test to run",
              multiple=True, default=[], type=click.UNPROCESSED)
@click.option("--test-filter", "test_filters",
              help="Run tests matching the given pattern",
              metavar="REGEXP", multiple=True, default=[],
              type=click.UNPROCESSED)
@click.option("--flags-to-test", "flags_to_test",
              help="Add a set of flags to test (space separated)",
              metavar="FLAGLIST", multiple=True, default=[],
              type=click.UNPROCESSED)
@click.option("--jobs-to-test", "jobs_to_test",
              help="Add a job count to test (full builds)",
              metavar="NUM", multiple=True, default=[], type=int)
@click.option("--config-to-test", "configs_to_test",
              help="Add build configuration to test (full builds)",
              metavar="NAME", multiple=True, default=[],
              type=click.Choice(['Debug', 'Release']))
#  Output Options
@click.option("--no-machdep-info", "use_machdep_info",
              help=("Don't put machine (instance) dependent "
                    "variables in machine info"),
              flag_value=False, default=True)
@click.option("--machine-name", "machine_name", type=click.UNPROCESSED,
              help="Machine name to use in submission",
              default=platform.uname()[1])
@click.option("--submit", "submit_url", metavar="URLORPATH",
              help=("autosubmit the test result to the given server "
                    "(or local instance)"),
              type=click.UNPROCESSED, default=None)
@submit_options
@click.option("--output", "output", metavar="PATH",
              help="write raw report data to PATH (or stdout if '-')")
@click.option("-v", "--verbose", "verbose",
              help="show verbose test results", is_flag=True)
def cli_action(*args, **kwargs):
    global opts

    compile_test = CompileTest()
    opts = compile_test.opts

    for key, value in kwargs.items():
        setattr(compile_test.opts, key, value)

    results = compile_test.run_test(compile_test.opts)
    compile_test.show_results_url(results)