1
0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2024-11-22 02:33:06 +01:00

[lit] Force site configs to be run before source-tree configs

This patch simplifies LLVM's lit infrastructure by enforcing an ordering
that a site config is always run before a source-tree config.

A significant amount of the complexity from lit config files arises from
the fact that inside of a source-tree config file, we don't yet know if
the site config has been run.  However it is *always* required to run
a site config first, because it passes various variables down through
CMake that the main config depends on.  As a result, every config
file has to do a bunch of magic to try to reverse-engineer the location
of the site config file if they detect (heuristically) that the site
config file has not yet been run.

This patch solves the problem by emitting a mapping from source tree
config file to binary tree site config file in llvm-lit.py. Then, during
discovery when we find a config file, we check to see if we have a
target mapping for it, and if so we use that instead.

This mechanism is generic enough that it does not affect external users
of lit. They will just not have a config mapping defined, and everything
will work as normal.

On the other hand, for us it allows us to make many simplifications:

* We are guaranteed that a site config will be executed first
* Inside of a main config, we no longer have to assume that attributes
  might not be present and use getattr everywhere.
* We no longer have to pass parameters such as --param llvm_site_config=<path>
  on the command line.
* It is future-proof, meaning you don't have to edit llvm-lit.in to add
  support for new projects.
* All of the duplicated logic of trying various fallback mechanisms of
  finding a site config from the main config are now gone.

One potentially noteworthy thing that was required to implement this
change is that whereas the ninja check targets previously used the first
method to spawn lit, they now use the second. In particular, you can no
longer run lit.py against the source tree while specifying the various
`foo_site_config=<path>` parameters.  Instead, you need to run
llvm-lit.py.

Differential Revision: https://reviews.llvm.org/D37756

llvm-svn: 313270
This commit is contained in:
Zachary Turner 2017-09-14 16:47:58 +00:00
parent 004d2b7e9e
commit 2fadf8bf73
8 changed files with 81 additions and 185 deletions

View File

@ -845,7 +845,6 @@ if( LLVM_INCLUDE_UTILS )
add_subdirectory(utils/PerfectShuffle)
add_subdirectory(utils/count)
add_subdirectory(utils/not)
add_subdirectory(utils/llvm-lit)
add_subdirectory(utils/yaml-bench)
else()
if ( LLVM_INCLUDE_TESTS )
@ -932,6 +931,11 @@ endif()
add_subdirectory(cmake/modules)
# Do this last so that all lit targets have already been created.
if (LLVM_INCLUDE_UTILS)
add_subdirectory(utils/llvm-lit)
endif()
if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
install(DIRECTORY include/llvm include/llvm-c
DESTINATION include

View File

@ -1174,6 +1174,13 @@ function(configure_lit_site_cfg input output)
endif()
configure_file(${input} ${output} @ONLY)
get_filename_component(INPUT_DIR ${input} DIRECTORY)
if (EXISTS "${INPUT_DIR}/lit.cfg")
set(PYTHON_STATEMENT "map_config('${INPUT_DIR}/lit.cfg', '${output}')")
get_property(LLVM_LIT_CONFIG_MAP GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP)
set(LLVM_LIT_CONFIG_MAP "${LLVM_LIT_CONFIG_MAP}\n${PYTHON_STATEMENT}")
set_property(GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP ${LLVM_LIT_CONFIG_MAP})
endif()
endfunction()
# A raw function to create a lit target. This is used to implement the testuite
@ -1185,12 +1192,16 @@ function(add_lit_target target comment)
if (NOT CMAKE_CFG_INTDIR STREQUAL ".")
list(APPEND LIT_ARGS --param build_mode=${CMAKE_CFG_INTDIR})
endif ()
if (EXISTS ${LLVM_MAIN_SRC_DIR}/utils/lit/lit.py)
set (LIT_COMMAND "${PYTHON_EXECUTABLE};${LLVM_MAIN_SRC_DIR}/utils/lit/lit.py"
CACHE STRING "Command used to spawn llvm-lit")
else()
find_program(LIT_COMMAND NAMES llvm-lit lit.py lit)
if (WIN32 AND NOT CYGWIN)
# llvm-lit needs suffix.py for multiprocess to find a main module.
set(suffix .py)
endif ()
set(llvm_lit_path ${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-lit${suffix})
set (LIT_COMMAND "${PYTHON_EXECUTABLE};${llvm_lit_path}"
CACHE STRING "Command used to spawn llvm-lit" FORCE)
list(APPEND LIT_COMMAND ${LIT_ARGS})
foreach(param ${ARG_PARAMS})
list(APPEND LIT_COMMAND --param ${param})

View File

@ -18,14 +18,11 @@ config.is_early = True
# test_source_root: The root path where tests are located.
# test_exec_root: The root path where tests should be run.
llvm_obj_root = getattr(config, 'llvm_obj_root', None)
if llvm_obj_root is not None:
config.test_exec_root = os.path.join(llvm_obj_root, 'unittests')
config.test_source_root = config.test_exec_root
config.test_exec_root = os.path.join(config.llvm_obj_root, 'unittests')
config.test_source_root = config.test_exec_root
# testFormat: The test format to use to interpret tests.
llvm_build_mode = getattr(config, 'llvm_build_mode', "Debug")
config.test_format = lit.formats.GoogleTest(llvm_build_mode, 'Tests')
config.test_format = lit.formats.GoogleTest(config.llvm_build_mode, 'Tests')
# Propagate the temp directory. Windows requires this because it uses \Windows\
# if none of these are present.
@ -47,49 +44,3 @@ if sys.platform in ['win32', 'cygwin'] and os.path.isdir(config.shlibdir):
# Win32 may use %SYSTEMDRIVE% during file system shell operations, so propogate.
if sys.platform == 'win32' and 'SYSTEMDRIVE' in os.environ:
config.environment['SYSTEMDRIVE'] = os.environ['SYSTEMDRIVE']
###
# Check that the object root is known.
if config.test_exec_root is None:
# Otherwise, we haven't loaded the site specific configuration (the user is
# probably trying to run on a test file directly, and either the site
# configuration hasn't been created by the build system, or we are in an
# out-of-tree build situation).
# Check for 'llvm_unit_site_config' user parameter, and use that if available.
site_cfg = lit_config.params.get('llvm_unit_site_config', None)
if site_cfg and os.path.exists(site_cfg):
lit_config.load_config(config, site_cfg)
raise SystemExit
# Try to detect the situation where we are using an out-of-tree build by
# looking for 'llvm-config'.
#
# FIXME: I debated (i.e., wrote and threw away) adding logic to
# automagically generate the lit.site.cfg if we are in some kind of fresh
# build situation. This means knowing how to invoke the build system
# though, and I decided it was too much magic.
llvm_config = lit.util.which('llvm-config', config.environment['PATH'])
if not llvm_config:
lit_config.fatal('No site specific configuration available!')
# Get the source and object roots.
llvm_src_root = subprocess.check_output(['llvm-config', '--src-root']).strip()
llvm_obj_root = subprocess.check_output(['llvm-config', '--obj-root']).strip()
# Validate that we got a tree which points to here.
this_src_root = os.path.join(os.path.dirname(__file__),'..','..')
if os.path.realpath(llvm_src_root) != os.path.realpath(this_src_root):
lit_config.fatal('No site specific configuration available!')
# Check that the site specific configuration exists.
site_cfg = os.path.join(llvm_obj_root, 'test', 'Unit', 'lit.site.cfg')
if not os.path.exists(site_cfg):
lit_config.fatal('No site specific configuration available!')
# Okay, that worked. Notify the user of the automagic, and reconfigure.
lit_config.note('using out-of-tree build at %r' % llvm_obj_root)
lit_config.load_config(config, site_cfg)
raise SystemExit

View File

@ -53,17 +53,11 @@ config.excludes = ['Inputs', 'CMakeLists.txt', 'README.txt', 'LICENSE.txt']
config.test_source_root = os.path.dirname(__file__)
# test_exec_root: The root path where tests should be run.
llvm_obj_root = getattr(config, 'llvm_obj_root', None)
if llvm_obj_root is not None:
config.test_exec_root = os.path.join(llvm_obj_root, 'test')
config.test_exec_root = os.path.join(config.llvm_obj_root, 'test')
# Tweak the PATH to include the tools dir.
if llvm_obj_root is not None:
llvm_tools_dir = getattr(config, 'llvm_tools_dir', None)
if not llvm_tools_dir:
lit_config.fatal('No LLVM tools dir set!')
path = os.path.pathsep.join((llvm_tools_dir, config.environment['PATH']))
config.environment['PATH'] = path
path = os.path.pathsep.join((config.llvm_tools_dir, config.environment['PATH']))
config.environment['PATH'] = path
# Propagate 'HOME' through the environment.
if 'HOME' in os.environ:
@ -85,7 +79,7 @@ if 'TEMP' in os.environ:
config.environment['TEMP'] = os.environ['TEMP']
# Propagate LLVM_SRC_ROOT into the environment.
config.environment['LLVM_SRC_ROOT'] = getattr(config, 'llvm_src_root', '')
config.environment['LLVM_SRC_ROOT'] = config.llvm_src_root
# Propagate PYTHON_EXECUTABLE into the environment
config.environment['PYTHON_EXECUTABLE'] = getattr(config, 'python_executable',
@ -97,80 +91,23 @@ for symbolizer in ['ASAN_SYMBOLIZER_PATH', 'MSAN_SYMBOLIZER_PATH']:
config.environment[symbolizer] = os.environ[symbolizer]
# Set up OCAMLPATH to include newly built OCaml libraries.
llvm_lib_dir = getattr(config, 'llvm_lib_dir', None)
if llvm_lib_dir is None:
if llvm_obj_root is not None:
llvm_lib_dir = os.path.join(llvm_obj_root, 'lib')
top_ocaml_lib = os.path.join(config.llvm_lib_dir, 'ocaml')
llvm_ocaml_lib = os.path.join(top_ocaml_lib, 'llvm')
ocamlpath = os.path.pathsep.join((llvm_ocaml_lib, top_ocaml_lib))
if 'OCAMLPATH' in os.environ:
ocamlpath = os.path.pathsep.join((ocamlpath, os.environ['OCAMLPATH']))
config.environment['OCAMLPATH'] = ocamlpath
if llvm_lib_dir is not None:
top_ocaml_lib = os.path.join(llvm_lib_dir, 'ocaml')
llvm_ocaml_lib = os.path.join(top_ocaml_lib, 'llvm')
if llvm_ocaml_lib is not None:
ocamlpath = os.path.pathsep.join((llvm_ocaml_lib, top_ocaml_lib))
if 'OCAMLPATH' in os.environ:
ocamlpath = os.path.pathsep.join((ocamlpath, os.environ['OCAMLPATH']))
config.environment['OCAMLPATH'] = ocamlpath
if 'CAML_LD_LIBRARY_PATH' in os.environ:
caml_ld_library_path = os.path.pathsep.join((llvm_ocaml_lib,
os.environ['CAML_LD_LIBRARY_PATH']))
config.environment['CAML_LD_LIBRARY_PATH'] = caml_ld_library_path
else:
config.environment['CAML_LD_LIBRARY_PATH'] = llvm_ocaml_lib
if 'CAML_LD_LIBRARY_PATH' in os.environ:
caml_ld_library_path = os.path.pathsep.join((llvm_ocaml_lib,
os.environ['CAML_LD_LIBRARY_PATH']))
config.environment['CAML_LD_LIBRARY_PATH'] = caml_ld_library_path
else:
config.environment['CAML_LD_LIBRARY_PATH'] = llvm_ocaml_lib
# Set up OCAMLRUNPARAM to enable backtraces in OCaml tests.
config.environment['OCAMLRUNPARAM'] = 'b'
###
import os
# Check that the object root is known.
if config.test_exec_root is None:
# Otherwise, we haven't loaded the site specific configuration (the user is
# probably trying to run on a test file directly, and either the site
# configuration hasn't been created by the build system, or we are in an
# out-of-tree build situation).
# Check for 'llvm_site_config' user parameter, and use that if available.
site_cfg = lit_config.params.get('llvm_site_config', None)
if site_cfg and os.path.exists(site_cfg):
lit_config.load_config(config, site_cfg)
raise SystemExit
# Try to detect the situation where we are using an out-of-tree build by
# looking for 'llvm-config'.
#
# FIXME: I debated (i.e., wrote and threw away) adding logic to
# automagically generate the lit.site.cfg if we are in some kind of fresh
# build situation. This means knowing how to invoke the build system
# though, and I decided it was too much magic.
llvm_config = lit.util.which('llvm-config', config.environment['PATH'])
if not llvm_config:
lit_config.fatal('No site specific configuration available!')
# Get the source and object roots.
llvm_src_root = subprocess.check_output(['llvm-config', '--src-root']).strip()
llvm_obj_root = subprocess.check_output(['llvm-config', '--obj-root']).strip()
# Validate that we got a tree which points to here.
this_src_root = os.path.dirname(config.test_source_root)
if os.path.realpath(llvm_src_root) != os.path.realpath(this_src_root):
lit_config.fatal('No site specific configuration available!')
# Check that the site specific configuration exists.
site_cfg = os.path.join(llvm_obj_root, 'test', 'lit.site.cfg')
if not os.path.exists(site_cfg):
lit_config.fatal('No site specific configuration available!')
# Okay, that worked. Notify the user of the automagic, and reconfigure.
lit_config.note('using out-of-tree build at %r' % llvm_obj_root)
lit_config.load_config(config, site_cfg)
raise SystemExit
###
# Provide the path to asan runtime lib 'libclang_rt.asan_osx_dynamic.dylib' if
# available. This is darwin specific since it's currently only needed on darwin.
def get_asan_rtlib():
@ -231,11 +168,11 @@ config.substitutions.append( ('%ld64', ld64_cmd) )
# Support tests for both native and bytecode builds.
config.substitutions.append( ('%ocamlc',
"%s ocamlc -cclib -L%s %s" %
(config.ocamlfind_executable, llvm_lib_dir, config.ocaml_flags)) )
(config.ocamlfind_executable, config.llvm_lib_dir, config.ocaml_flags)) )
if config.have_ocamlopt:
config.substitutions.append( ('%ocamlopt',
"%s ocamlopt -cclib -L%s -cclib -Wl,-rpath,%s %s" %
(config.ocamlfind_executable, llvm_lib_dir, llvm_lib_dir, config.ocaml_flags)) )
(config.ocamlfind_executable, config.llvm_lib_dir, config.llvm_lib_dir, config.ocaml_flags)) )
else:
config.substitutions.append( ('%ocamlopt', "true" ) )
@ -267,7 +204,7 @@ def find_tool_substitution(pattern):
# llvm-lit "-Dllc=llc -enable-misched -verify-machineinstrs"
tool_path = lit_config.params.get(tool_name)
if tool_path is None:
tool_path = lit.util.which(tool_name, llvm_tools_dir)
tool_path = lit.util.which(tool_name, config.llvm_tools_dir)
if tool_path is None:
return tool_name, tool_path, tool_pipe
if (tool_name == "llc" and
@ -333,8 +270,8 @@ for pattern in [r"\bbugpoint\b(?!-)",
tool_name, tool_path, tool_pipe = find_tool_substitution(pattern)
if not tool_path:
# Warn, but still provide a substitution.
lit_config.note('Did not find ' + tool_name + ' in ' + llvm_tools_dir)
tool_path = llvm_tools_dir + '/' + tool_name
lit_config.note('Did not find ' + tool_name + ' in ' + config.llvm_tools_dir)
tool_path = config.llvm_tools_dir + '/' + tool_name
config.substitutions.append((pattern, tool_pipe + tool_path))
# For tools that are optional depending on the config, we won't warn
@ -350,7 +287,7 @@ for pattern in [r"\bllvm-go\b",
tool_name, tool_path, tool_pipe = find_tool_substitution(pattern)
if not tool_path:
# Provide a substitution anyway, for the sake of consistent errors.
tool_path = llvm_tools_dir + '/' + tool_name
tool_path = config.llvm_tools_dir + '/' + tool_name
config.substitutions.append((pattern, tool_pipe + tool_path))
@ -482,11 +419,11 @@ if have_ld64_plugin_support():
# Ask llvm-config about assertion mode.
try:
llvm_config_cmd = subprocess.Popen(
[os.path.join(llvm_tools_dir, 'llvm-config'), '--assertion-mode'],
[os.path.join(config.llvm_tools_dir, 'llvm-config'), '--assertion-mode'],
stdout = subprocess.PIPE,
env=config.environment)
except OSError:
print("Could not find llvm-config in " + llvm_tools_dir)
print("Could not find llvm-config in " + config.llvm_tools_dir)
exit(42)
if re.search(r'ON', llvm_config_cmd.stdout.read().decode('ascii')):
@ -538,11 +475,11 @@ if use_gmalloc:
# Ask llvm-config about global-isel.
try:
llvm_config_cmd = subprocess.Popen(
[os.path.join(llvm_tools_dir, 'llvm-config'), '--has-global-isel'],
[os.path.join(config.llvm_tools_dir, 'llvm-config'), '--has-global-isel'],
stdout = subprocess.PIPE,
env=config.environment)
except OSError:
print("Could not find llvm-config in " + llvm_tools_dir)
print("Could not find llvm-config in " + config.llvm_tools_dir)
exit(42)
if re.search(r'ON', llvm_config_cmd.stdout.read().decode('ascii')):

View File

@ -40,6 +40,20 @@ def getTestSuite(item, litConfig, cache):
ts, relative = search(parent)
return (ts, relative + (base,))
# This is a private builtin parameter which can be used to perform
# translation of configuration paths. Specifically, this parameter
# can be set to a dictionary that the discovery process will consult
# when it finds a configuration it is about to load. If the given
# path is in the map, the value of that key is a path to the
# configuration to load instead.
config_map = litConfig.params.get('config_map')
if config_map:
cfgpath = os.path.normpath(cfgpath)
cfgpath = os.path.normcase(cfgpath)
target = config_map.get(cfgpath)
if target:
cfgpath = target
# We found a test suite, create a new config for it and load it.
if litConfig.debug:
litConfig.note('loading suite config %r' % cfgpath)
@ -212,7 +226,7 @@ def find_tests_for_inputs(lit_config, inputs):
f.close()
else:
actual_inputs.append(input)
# Load the tests from the inputs.
tests = []
test_suite_cache = {}

View File

@ -65,7 +65,6 @@ if sys.platform.startswith('win') or sys.platform.startswith('cygwin'):
config.available_features.add('windows')
# Add llvm tools directory if this config is being loaded indirectly
llvm_tools_dir = getattr(config, 'llvm_tools_dir', None)
if llvm_tools_dir != None:
path = os.path.pathsep.join((llvm_tools_dir, config.environment['PATH']))
if config.llvm_tools_dir is not None:
path = os.path.pathsep.join((config.llvm_tools_dir, config.environment['PATH']))
config.environment['PATH'] = path

View File

@ -4,6 +4,8 @@ if (WIN32 AND NOT CYGWIN)
endif ()
set(llvm_lit_path ${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-lit${suffix})
get_property(LLVM_LIT_CONFIG_MAP GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP)
if(NOT "${CMAKE_CFG_INTDIR}" STREQUAL ".")
foreach(BUILD_MODE ${CMAKE_CONFIGURATION_TYPES})
string(REPLACE ${CMAKE_CFG_INTDIR} ${BUILD_MODE} bi ${llvm_lit_path})

View File

@ -3,6 +3,15 @@
import os
import sys
config_map = {}
def map_config(source_dir, site_config):
global config_map
source_dir = os.path.normpath(source_dir)
source_dir = os.path.normcase(source_dir)
site_config = os.path.normpath(site_config)
config_map[source_dir] = site_config
# Variables configured at build time.
llvm_source_root = "@LLVM_SOURCE_DIR@"
llvm_obj_root = "@LLVM_BINARY_DIR@"
@ -12,42 +21,11 @@ sys.path.insert(0, os.path.join(llvm_source_root, 'utils', 'lit'))
# Set up some builtin parameters, so that by default the LLVM test suite
# configuration file knows how to find the object tree.
builtin_parameters = {
'build_mode' : "@BUILD_MODE@",
'llvm_site_config' : os.path.join(llvm_obj_root, 'test', 'lit.site.cfg'),
'llvm_unit_site_config' : os.path.join(llvm_obj_root, 'test', 'Unit',
'lit.site.cfg')
}
builtin_parameters = { 'build_mode' : "@BUILD_MODE@" }
clang_obj_root = os.path.join(llvm_obj_root, 'tools', 'clang')
@LLVM_LIT_CONFIG_MAP@
if os.path.exists(clang_obj_root):
builtin_parameters['clang_site_config'] = \
os.path.join(clang_obj_root, 'test', 'lit.site.cfg')
clang_tools_extra_obj_root = os.path.join(clang_obj_root, 'tools', 'extra')
if os.path.exists(clang_tools_extra_obj_root):
builtin_parameters['clang_tools_extra_site_config'] = \
os.path.join(clang_tools_extra_obj_root, 'test', 'lit.site.cfg')
lld_obj_root = os.path.join(llvm_obj_root, 'tools', 'lld')
if os.path.exists(lld_obj_root):
builtin_parameters['lld_site_config'] = \
os.path.join(lld_obj_root, 'test', 'lit.site.cfg')
compilerrt_obj_root = os.path.join(llvm_obj_root, 'projects', 'compiler-rt')
if os.path.exists(compilerrt_obj_root):
builtin_parameters['compilerrt_site_basedir'] = \
os.path.join(compilerrt_obj_root, 'test')
libcxx_obj_root = os.path.join(llvm_obj_root, 'projects', 'libcxx')
if os.path.exists(libcxx_obj_root):
builtin_parameters['libcxx_site_config'] = \
os.path.join(libcxx_obj_root, 'test', 'lit.site.cfg')
libcxxabi_obj_root = os.path.join(llvm_obj_root, 'projects', 'libcxxabi')
if os.path.exists(libcxxabi_obj_root):
builtin_parameters['libcxxabi_site_config'] = \
os.path.join(libcxxabi_obj_root, 'test', 'lit.site.cfg')
builtin_parameters['config_map'] = config_map
if __name__=='__main__':
from lit.main import main