diff options
author | Chandler Carruth <chandlerc@gmail.com> | 2012-08-07 07:09:14 +0000 |
---|---|---|
committer | Chandler Carruth <chandlerc@gmail.com> | 2012-08-07 07:09:14 +0000 |
commit | 5f8cfa76e86acf74d3656b82476995a8fa16a6b5 (patch) | |
tree | e2a391cd393045645e86f3b902c1a502ded55a6e /test | |
parent | 85a175a9b44b02fdf258c0325604a2c23355867c (diff) |
Copy some of the testing infrastructure from the primary Clang repository into
the new tools repo to preserve history. I'll subsequently be editting these
down to work w/ the tooling test suites.
git-svn-id: https://llvm.org/svn/llvm-project/clang-tools-extra/trunk@161402 91177308-0d34-0410-b5e6-96231b3b80d8
Diffstat (limited to 'test')
-rw-r--r-- | test/CMakeLists.txt | 93 | ||||
-rw-r--r-- | test/Makefile | 71 | ||||
-rw-r--r-- | test/lit.cfg | 261 | ||||
-rw-r--r-- | test/lit.site.cfg.in | 21 |
4 files changed, 446 insertions, 0 deletions
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt new file mode 100644 index 00000000..8184c3d3 --- /dev/null +++ b/test/CMakeLists.txt @@ -0,0 +1,93 @@ +# Test runner infrastructure for Clang. This configures the Clang test trees +# for use by Lit, and delegates to LLVM's lit test handlers. +# +# If this is a stand-alone Clang build, we fake up our own Lit support here +# rather than relying on LLVM's. + +set(CLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/..") +set(CLANG_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/..") + +configure_lit_site_cfg( + ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in + ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg + ) + +configure_lit_site_cfg( + ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.in + ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg + ) + +if( PATH_TO_LLVM_BUILD ) + set(CLANG_TEST_EXTRA_ARGS "--path=${CLANG_BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}") +endif() + +option(CLANG_TEST_USE_VG "Run Clang tests under Valgrind" OFF) +if(CLANG_TEST_USE_VG) + set(CLANG_TEST_EXTRA_ARGS ${CLANG_TEST_EXTRA_ARGS} "--vg") +endif () + +if( NOT CLANG_BUILT_STANDALONE ) + + set(CLANG_TEST_DEPS + clang clang-headers + c-index-test diagtool arcmt-test c-arcmt-test + clang-check + llvm-dis llc opt FileCheck count not + ) + set(CLANG_TEST_PARAMS + clang_site_config=${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg + ) + + if(LLVM_INCLUDE_TESTS) + list(APPEND CLANG_TEST_DEPS ClangUnitTests) + list(APPEND CLANG_TEST_PARAMS + clang_unit_site_config=${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg + ) + endif() + add_lit_testsuite(check-clang "Running the Clang regression tests" + ${CMAKE_CURRENT_BINARY_DIR} + PARAMS ${CLANG_TEST_PARAMS} + DEPENDS ${CLANG_TEST_DEPS} + ARGS ${CLANG_TEST_EXTRA_ARGS} + ) + set_target_properties(check-clang PROPERTIES FOLDER "Clang tests") + +else() + + include(FindPythonInterp) + if(PYTHONINTERP_FOUND) + if( LLVM_MAIN_SRC_DIR ) + set(LIT "${LLVM_MAIN_SRC_DIR}/utils/lit/lit.py") + else() + set(LIT "${PATH_TO_LLVM_BUILD}/bin/${CMAKE_CFG_INTDIR}/llvm-lit") + # Installed LLVM does not contain ${CMAKE_CFG_INTDIR} in paths. + if( NOT EXISTS ${LIT} ) + set(LIT "${PATH_TO_LLVM_BUILD}/bin/llvm-lit") + endif() + endif() + + set(LIT_ARGS "${CLANG_TEST_EXTRA_ARGS} ${LLVM_LIT_ARGS}") + separate_arguments(LIT_ARGS) + + add_custom_target(check-clang + COMMAND ${PYTHON_EXECUTABLE} + ${LIT} + --param clang_site_config=${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg + --param build_config=${CMAKE_CFG_INTDIR} + --param build_mode=${RUNTIME_BUILD_MODE} + ${LIT_ARGS} + ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Running Clang regression tests" + DEPENDS clang clang-headers + c-index-test diagtool arcmt-test c-arcmt-test + clang-check + ) + set_target_properties(check-clang PROPERTIES FOLDER "Clang tests") + endif() + +endif() + +# Add a legacy target spelling: clang-test +add_custom_target(clang-test) +add_dependencies(clang-test check-clang) +set_target_properties(clang-test PROPERTIES FOLDER "Clang tests") diff --git a/test/Makefile b/test/Makefile new file mode 100644 index 00000000..4fdafe14 --- /dev/null +++ b/test/Makefile @@ -0,0 +1,71 @@ +CLANG_LEVEL := .. +include $(CLANG_LEVEL)/Makefile + +# Test in all immediate subdirectories if unset. +ifdef TESTSUITE +TESTDIRS := $(TESTSUITE:%=$(PROJ_SRC_DIR)/%) +else +TESTDIRS ?= $(PROJ_SRC_DIR) +endif + +# 'lit' wants objdir paths, so it will pick up the lit.site.cfg. +TESTDIRS := $(TESTDIRS:$(PROJ_SRC_DIR)%=$(PROJ_OBJ_DIR)%) + +# Allow EXTRA_TESTDIRS to provide additional test directories. +TESTDIRS += $(EXTRA_TESTDIRS) + +ifndef TESTARGS +ifdef VERBOSE +TESTARGS = -v +else +TESTARGS = -s -v +endif +endif + +# Make sure any extra test suites can find the main site config. +LIT_ARGS := --param clang_site_config=$(PROJ_OBJ_DIR)/lit.site.cfg + +ifdef VG + LIT_ARGS += "--vg" +endif + +all:: lit.site.cfg Unit/lit.site.cfg + @ echo '--- Running clang tests for $(TARGET_TRIPLE) ---' + @ $(PYTHON) $(LLVM_SRC_ROOT)/utils/lit/lit.py \ + $(LIT_ARGS) $(TESTARGS) $(TESTDIRS) + +FORCE: + +lit.site.cfg: FORCE + @echo "Making Clang 'lit.site.cfg' file..." + @$(ECHOPATH) s=@LLVM_SOURCE_DIR@=$(LLVM_SRC_ROOT)=g > lit.tmp + @$(ECHOPATH) s=@LLVM_BINARY_DIR@=$(LLVM_OBJ_ROOT)=g >> lit.tmp + @$(ECHOPATH) s=@LLVM_TOOLS_DIR@=$(ToolDir)=g >> lit.tmp + @$(ECHOPATH) s=@LLVM_LIBS_DIR@=$(LibDir)=g >> lit.tmp + @$(ECHOPATH) s=@CLANG_SOURCE_DIR@=$(PROJ_SRC_DIR)/..=g >> lit.tmp + @$(ECHOPATH) s=@CLANG_BINARY_DIR@=$(PROJ_OBJ_DIR)/..=g >> lit.tmp + @$(ECHOPATH) s=@TARGET_TRIPLE@=$(TARGET_TRIPLE)=g >> lit.tmp + @sed -f lit.tmp $(PROJ_SRC_DIR)/lit.site.cfg.in > $@ + @-rm -f lit.tmp + +Unit/lit.site.cfg: FORCE + @echo "Making Clang 'Unit/lit.site.cfg' file..." + @$(MKDIR) $(dir $@) + @$(ECHOPATH) s=@LLVM_SOURCE_DIR@=$(LLVM_SRC_ROOT)=g > unit.tmp + @$(ECHOPATH) s=@LLVM_BINARY_DIR@=$(LLVM_OBJ_ROOT)=g >> unit.tmp + @$(ECHOPATH) s=@LLVM_TOOLS_DIR@=$(ToolDir)=g >> unit.tmp + @$(ECHOPATH) s=@LLVM_LIBS_DIR@=$(LibDir)=g >> unit.tmp + @$(ECHOPATH) s=@CLANG_SOURCE_DIR@=$(PROJ_SRC_DIR)/..=g >> unit.tmp + @$(ECHOPATH) s=@CLANG_BINARY_DIR@=$(PROJ_OBJ_DIR)/..=g >> unit.tmp + @$(ECHOPATH) s=@TARGET_TRIPLE@=$(TARGET_TRIPLE)=g >> unit.tmp + @$(ECHOPATH) s=@LLVM_BUILD_MODE@=$(BuildMode)=g >> unit.tmp + @$(ECHOPATH) s=@ENABLE_SHARED@=$(ENABLE_SHARED)=g >> unit.tmp + @$(ECHOPATH) s=@SHLIBDIR@=$(SharedLibDir)=g >> unit.tmp + @$(ECHOPATH) s=@SHLIBPATH_VAR@=$(SHLIBPATH_VAR)=g >> unit.tmp + @sed -f unit.tmp $(PROJ_SRC_DIR)/Unit/lit.site.cfg.in > $@ + @-rm -f unit.tmp + +clean:: + @ find . -name Output | xargs rm -fr + +.PHONY: all report clean diff --git a/test/lit.cfg b/test/lit.cfg new file mode 100644 index 00000000..1fc60593 --- /dev/null +++ b/test/lit.cfg @@ -0,0 +1,261 @@ +# -*- Python -*- + +import os +import platform +import re +import subprocess + + +# Configuration file for the 'lit' test runner. + +# name: The name of this test suite. +config.name = 'Clang' + +# Tweak PATH for Win32 +if platform.system() == 'Windows': + # Seek sane tools in directories and set to $PATH. + path = getattr(config, 'lit_tools_dir', None) + path = lit.getToolsPath(path, + config.environment['PATH'], + ['cmp.exe', 'grep.exe', 'sed.exe']) + if path is not None: + path = os.path.pathsep.join((path, + config.environment['PATH'])) + config.environment['PATH'] = path + +# testFormat: The test format to use to interpret tests. +# +# For now we require '&&' between commands, until they get globally killed and +# the test runner updated. +execute_external = (platform.system() != 'Windows' + or lit.getBashPath() not in [None, ""]) +config.test_format = lit.formats.ShTest(execute_external) + +# suffixes: A list of file extensions to treat as test files. +config.suffixes = ['.c', '.cpp', '.m', '.mm', '.cu', '.ll', '.cl', '.s'] + +# test_source_root: The root path where tests are located. +config.test_source_root = os.path.dirname(__file__) + +# test_exec_root: The root path where tests should be run. +clang_obj_root = getattr(config, 'clang_obj_root', None) +if clang_obj_root is not None: + config.test_exec_root = os.path.join(clang_obj_root, 'test') + +# Set llvm_{src,obj}_root for use by others. +config.llvm_src_root = getattr(config, 'llvm_src_root', None) +config.llvm_obj_root = getattr(config, 'llvm_obj_root', None) + +# Clear some environment variables that might affect Clang. +# +# This first set of vars are read by Clang, but shouldn't affect tests +# that aren't specifically looking for these features, or are required +# simply to run the tests at all. +# +# FIXME: Should we have a tool that enforces this? + +# safe_env_vars = ('TMPDIR', 'TEMP', 'TMP', 'USERPROFILE', 'PWD', +# 'MACOSX_DEPLOYMENT_TARGET', 'IPHONEOS_DEPLOYMENT_TARGET', +# 'IOS_SIMULATOR_DEPLOYMENT_TARGET', +# 'VCINSTALLDIR', 'VC100COMNTOOLS', 'VC90COMNTOOLS', +# 'VC80COMNTOOLS') +possibly_dangerous_env_vars = ['COMPILER_PATH', 'RC_DEBUG_OPTIONS', + 'CINDEXTEST_PREAMBLE_FILE', 'LIBRARY_PATH', + 'CPATH', 'C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH', + 'OBJC_INCLUDE_PATH', 'OBJCPLUS_INCLUDE_PATH', + 'LIBCLANG_TIMING', 'LIBCLANG_OBJTRACKING', + 'LIBCLANG_LOGGING', 'LIBCLANG_BGPRIO_INDEX', + 'LIBCLANG_BGPRIO_EDIT', 'LIBCLANG_NOTHREADS', + 'LIBCLANG_RESOURCE_USAGE', + 'LIBCLANG_CODE_COMPLETION_LOGGING'] +# Clang/Win32 may refer to %INCLUDE%. vsvarsall.bat sets it. +if platform.system() != 'Windows': + possibly_dangerous_env_vars.append('INCLUDE') +for name in possibly_dangerous_env_vars: + if name in config.environment: + del config.environment[name] + +# Tweak the PATH to include the tools dir and the scripts dir. +if clang_obj_root is not None: + llvm_tools_dir = getattr(config, 'llvm_tools_dir', None) + if not llvm_tools_dir: + lit.fatal('No LLVM tools dir set!') + path = os.path.pathsep.join((llvm_tools_dir, config.environment['PATH'])) + config.environment['PATH'] = path + + llvm_libs_dir = getattr(config, 'llvm_libs_dir', None) + if not llvm_libs_dir: + lit.fatal('No LLVM libs dir set!') + path = os.path.pathsep.join((llvm_libs_dir, + config.environment.get('LD_LIBRARY_PATH',''))) + config.environment['LD_LIBRARY_PATH'] = path + +### + +# Check that the object root is known. +if config.test_exec_root is None: + # Otherwise, we haven't loaded the site specific configuration (the user is + # probably trying to run on a test file directly, and either the site + # configuration hasn't been created by the build system, or we are in an + # out-of-tree build situation). + + # Check for 'clang_site_config' user parameter, and use that if available. + site_cfg = lit.params.get('clang_site_config', None) + if site_cfg and os.path.exists(site_cfg): + lit.load_config(config, site_cfg) + raise SystemExit + + # Try to detect the situation where we are using an out-of-tree build by + # looking for 'llvm-config'. + # + # FIXME: I debated (i.e., wrote and threw away) adding logic to + # automagically generate the lit.site.cfg if we are in some kind of fresh + # build situation. This means knowing how to invoke the build system though, + # and I decided it was too much magic. We should solve this by just having + # the .cfg files generated during the configuration step. + + llvm_config = lit.util.which('llvm-config', config.environment['PATH']) + if not llvm_config: + lit.fatal('No site specific configuration available!') + + # Get the source and object roots. + llvm_src_root = lit.util.capture(['llvm-config', '--src-root']).strip() + llvm_obj_root = lit.util.capture(['llvm-config', '--obj-root']).strip() + clang_src_root = os.path.join(llvm_src_root, "tools", "clang") + clang_obj_root = os.path.join(llvm_obj_root, "tools", "clang") + + # Validate that we got a tree which points to here, using the standard + # tools/clang layout. + this_src_root = os.path.dirname(config.test_source_root) + if os.path.realpath(clang_src_root) != os.path.realpath(this_src_root): + lit.fatal('No site specific configuration available!') + + # Check that the site specific configuration exists. + site_cfg = os.path.join(clang_obj_root, 'test', 'lit.site.cfg') + if not os.path.exists(site_cfg): + lit.fatal('No site specific configuration available! You may need to ' + 'run "make test" in your Clang build directory.') + + # Okay, that worked. Notify the user of the automagic, and reconfigure. + lit.note('using out-of-tree build at %r' % clang_obj_root) + lit.load_config(config, site_cfg) + raise SystemExit + +### + +# Discover the 'clang' and 'clangcc' to use. + +import os + +def inferClang(PATH): + # Determine which clang to use. + clang = os.getenv('CLANG') + + # If the user set clang in the environment, definitely use that and don't + # try to validate. + if clang: + return clang + + # Otherwise look in the path. + clang = lit.util.which('clang', PATH) + + if not clang: + lit.fatal("couldn't find 'clang' program, try setting " + "CLANG in your environment") + + return clang + +# When running under valgrind, we mangle '-vg' onto the end of the triple so we +# can check it with XFAIL and XTARGET. +if lit.useValgrind: + config.target_triple += '-vg' + +config.clang = inferClang(config.environment['PATH']).replace('\\', '/') +if not lit.quiet: + lit.note('using clang: %r' % config.clang) + +# Note that when substituting %clang_cc1 also fill in the include directory of +# the builtin headers. Those are part of even a freestanding environment, but +# Clang relies on the driver to locate them. +def getClangBuiltinIncludeDir(clang): + # FIXME: Rather than just getting the version, we should have clang print + # out its resource dir here in an easy to scrape form. + cmd = subprocess.Popen([clang, '-print-file-name=include'], + stdout=subprocess.PIPE) + if not cmd.stdout: + lit.fatal("Couldn't find the include dir for Clang ('%s')" % clang) + return cmd.stdout.read().strip() + +config.substitutions.append( ('%clang_cc1', '%s -cc1 -internal-isystem %s' + % (config.clang, + getClangBuiltinIncludeDir(config.clang))) ) + +config.substitutions.append( ('%clangxx', ' ' + config.clang + + ' -ccc-clang-cxx -ccc-cxx ')) +config.substitutions.append( ('%clang', ' ' + config.clang + ' ') ) +config.substitutions.append( ('%test_debuginfo', ' ' + config.llvm_src_root + '/utils/test_debuginfo.pl ') ) + +# FIXME: Find nicer way to prohibit this. +config.substitutions.append( + (' clang ', """*** Do not use 'clang' in tests, use '%clang'. ***""") ) +config.substitutions.append( + (' clang\+\+ ', """*** Do not use 'clang++' in tests, use '%clangxx'. ***""")) +config.substitutions.append( + (' clang-cc ', + """*** Do not use 'clang-cc' in tests, use '%clang_cc1'. ***""") ) +config.substitutions.append( + (' clang -cc1 ', + """*** Do not use 'clang -cc1' in tests, use '%clang_cc1'. ***""") ) +config.substitutions.append( + (' %clang-cc1 ', + """*** invalid substitution, use '%clang_cc1'. ***""") ) + +### + +# Set available features we allow tests to conditionalize on. +# +# As of 2011.08, crash-recovery tests still do not pass on FreeBSD. +if platform.system() not in ['FreeBSD']: + config.available_features.add('crash-recovery') + +# Shell execution +if platform.system() not in ['Windows'] or lit.getBashPath() != '': + config.available_features.add('shell') + +# ANSI escape sequences in non-dump terminal +if platform.system() not in ['Windows']: + config.available_features.add('ansi-escape-sequences') + +# Registered Targets +def get_llc_props(tool): + set_of_targets = set() + enable_assertions = False + + cmd = subprocess.Popen([tool, '-version'], stdout=subprocess.PIPE) + + # Parse the stdout to get the list of registered targets. + parse_targets = False + for line in cmd.stdout: + if parse_targets: + m = re.match( r'(.*) - ', line) + if m is not None: + set_of_targets.add(m.group(1).strip() + '-registered-target') + else: + break + elif "Registered Targets:" in line: + parse_targets = True + + if re.search(r'with assertions', line): + enable_assertions = True + + return {"set_of_targets": set_of_targets, + "enable_assertions": enable_assertions} + +llc_props = get_llc_props(os.path.join(llvm_tools_dir, 'llc')) +if len(llc_props['set_of_targets']) > 0: + config.available_features.update(llc_props['set_of_targets']) +else: + lit.fatal('No Targets Registered with the LLVM Tools!') + +if llc_props['enable_assertions']: + config.available_features.add('asserts') diff --git a/test/lit.site.cfg.in b/test/lit.site.cfg.in new file mode 100644 index 00000000..df90b810 --- /dev/null +++ b/test/lit.site.cfg.in @@ -0,0 +1,21 @@ +## Autogenerated by LLVM/Clang configuration. +# Do not edit! +config.llvm_src_root = "@LLVM_SOURCE_DIR@" +config.llvm_obj_root = "@LLVM_BINARY_DIR@" +config.llvm_tools_dir = "@LLVM_TOOLS_DIR@" +config.llvm_libs_dir = "@LLVM_LIBS_DIR@" +config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@" +config.clang_obj_root = "@CLANG_BINARY_DIR@" +config.target_triple = "@TARGET_TRIPLE@" + +# Support substitution of the tools and libs dirs with user parameters. This is +# used when we can't determine the tool dir at configuration time. +try: + config.llvm_tools_dir = config.llvm_tools_dir % lit.params + config.llvm_libs_dir = config.llvm_libs_dir % lit.params +except KeyError,e: + key, = e.args + lit.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key)) + +# Let the main config do the real work. +lit.load_config(config, "@CLANG_SOURCE_DIR@/test/lit.cfg") |