[llvm] r313928 - Resubmit "[lit] Refactor out some more common lit configuration code."
Zachary Turner via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 21 15:16:40 PDT 2017
Author: zturner
Date: Thu Sep 21 15:16:40 2017
New Revision: 313928
URL: http://llvm.org/viewvc/llvm-project?rev=313928&view=rev
Log:
Resubmit "[lit] Refactor out some more common lit configuration code."
There were two issues, one Python 3 specific related to Unicode,
and another which is that the tool substitution for lld no longer
rejected matches where a / preceded the tool name.
Modified:
llvm/trunk/test/lit.cfg.py
llvm/trunk/utils/lit/lit/llvm/__init__.py
llvm/trunk/utils/lit/lit/llvm/config.py
llvm/trunk/utils/lit/lit/util.py
Modified: llvm/trunk/test/lit.cfg.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/lit.cfg.py?rev=313928&r1=313927&r2=313928&view=diff
==============================================================================
--- llvm/trunk/test/lit.cfg.py (original)
+++ llvm/trunk/test/lit.cfg.py Thu Sep 21 15:16:40 2017
@@ -11,6 +11,7 @@ import subprocess
import lit.util
import lit.formats
from lit.llvm import llvm_config
+from lit.llvm import ToolFilter
# name: The name of this test suite.
config.name = 'LLVM'
@@ -134,108 +135,39 @@ else:
# The regex is a pre-assertion to avoid matching a preceding
# dot, hyphen, carat, or slash (.foo, -foo, etc.). Some patterns
# also have a post-assertion to not match a trailing hyphen (foo-).
-NOJUNK = r"(?<!\.|-|\^|/|<)"
+JUNKCHARS = r".-^/<"
+required_tools = [
+ 'lli', 'llvm-ar', 'llvm-as', 'llvm-bcanalyzer', 'llvm-config', 'llvm-cov',
+ 'llvm-cxxdump', 'llvm-cvtres', 'llvm-diff', 'llvm-dis', 'llvm-dsymutil',
+ 'llvm-dwarfdump', 'llvm-extract', 'llvm-isel-fuzzer', 'llvm-lib',
+ 'llvm-link', 'llvm-lto', 'llvm-lto2', 'llvm-mc', 'llvm-mcmarkup',
+ 'llvm-modextract', 'llvm-nm', 'llvm-objcopy', 'llvm-objdump',
+ 'llvm-pdbutil', 'llvm-profdata', 'llvm-ranlib', 'llvm-readobj',
+ 'llvm-rtdyld', 'llvm-size', 'llvm-split', 'llvm-strings', 'llvm-tblgen',
+ 'llvm-c-test', 'llvm-cxxfilt', 'llvm-xray', 'yaml2obj', 'obj2yaml',
+ 'FileCheck', 'yaml-bench', 'verify-uselistorder',
+ ToolFilter('bugpoint', post='-'),
+ ToolFilter('llc', pre=JUNKCHARS),
+ ToolFilter('llvm-symbolizer', pre=JUNKCHARS),
+ ToolFilter('opt', JUNKCHARS),
+ ToolFilter('sancov', pre=JUNKCHARS),
+ ToolFilter('sanstats', pre=JUNKCHARS),
+ # Handle these specially as they are strings searched for during testing.
+ ToolFilter(r'\| \bcount\b', verbatim=True),
+ ToolFilter(r'\| \bnot\b', verbatim=True)]
-def find_tool_substitution(pattern):
- # Extract the tool name from the pattern. This relies on the tool
- # name being surrounded by \b word match operators. If the
- # pattern starts with "| ", include it in the string to be
- # substituted.
- tool_match = re.match(r"^(\\)?((\| )?)\W+b([0-9A-Za-z-_]+)\\b\W*$",
- pattern)
- tool_pipe = tool_match.group(2)
- tool_name = tool_match.group(4)
- # Did the user specify the tool path + arguments? This allows things like
- # llvm-lit "-Dllc=llc -enable-misched -verify-machineinstrs"
- tool_path = lit_config.params.get(tool_name)
- if tool_path is None:
- tool_path = lit.util.which(tool_name, config.llvm_tools_dir)
- if tool_path is None:
- return tool_name, tool_path, tool_pipe
- if (tool_name == "llc" and
- 'LLVM_ENABLE_MACHINE_VERIFIER' in os.environ and
- os.environ['LLVM_ENABLE_MACHINE_VERIFIER'] == "1"):
- tool_path += " -verify-machineinstrs"
- if (tool_name == "llvm-go"):
- tool_path += " go=" + config.go_executable
- return tool_name, tool_path, tool_pipe
-
-
-for pattern in [r"\bbugpoint\b(?!-)",
- NOJUNK + r"\bllc\b",
- r"\blli\b",
- r"\bllvm-ar\b",
- r"\bllvm-as\b",
- r"\bllvm-bcanalyzer\b",
- r"\bllvm-config\b",
- r"\bllvm-cov\b",
- r"\bllvm-cxxdump\b",
- r"\bllvm-cvtres\b",
- r"\bllvm-diff\b",
- r"\bllvm-dis\b",
- r"\bllvm-dsymutil\b",
- r"\bllvm-dwarfdump\b",
- r"\bllvm-extract\b",
- r"\bllvm-isel-fuzzer\b",
- r"\bllvm-lib\b",
- r"\bllvm-link\b",
- r"\bllvm-lto\b",
- r"\bllvm-lto2\b",
- r"\bllvm-mc\b",
- r"\bllvm-mcmarkup\b",
- r"\bllvm-modextract\b",
- r"\bllvm-nm\b",
- r"\bllvm-objcopy\b",
- r"\bllvm-objdump\b",
- r"\bllvm-pdbutil\b",
- r"\bllvm-profdata\b",
- r"\bllvm-ranlib\b",
- r"\bllvm-readobj\b",
- r"\bllvm-rtdyld\b",
- r"\bllvm-size\b",
- r"\bllvm-split\b",
- r"\bllvm-strings\b",
- r"\bllvm-tblgen\b",
- r"\bllvm-c-test\b",
- r"\bllvm-cxxfilt\b",
- r"\bllvm-xray\b",
- NOJUNK + r"\bllvm-symbolizer\b",
- NOJUNK + r"\bopt\b",
- r"\bFileCheck\b",
- r"\bobj2yaml\b",
- NOJUNK + r"\bsancov\b",
- NOJUNK + r"\bsanstats\b",
- r"\byaml2obj\b",
- r"\byaml-bench\b",
- r"\bverify-uselistorder\b",
- # Handle these specially as they are strings searched
- # for during testing.
- r"\| \bcount\b",
- r"\| \bnot\b"]:
- tool_name, tool_path, tool_pipe = find_tool_substitution(pattern)
- if not tool_path:
- # Warn, but still provide a substitution.
- lit_config.note('Did not find ' + tool_name + ' in ' + config.llvm_tools_dir)
- tool_path = config.llvm_tools_dir + '/' + tool_name
- config.substitutions.append((pattern, tool_pipe + tool_path))
+llvm_config.add_tool_substitutions(required_tools, config.llvm_tools_dir)
# For tools that are optional depending on the config, we won't warn
# if they're missing.
-for pattern in [r"\bllvm-go\b",
- r"\bllvm-mt\b",
- r"\bKaleidoscope-Ch3\b",
- r"\bKaleidoscope-Ch4\b",
- r"\bKaleidoscope-Ch5\b",
- r"\bKaleidoscope-Ch6\b",
- r"\bKaleidoscope-Ch7\b",
- r"\bKaleidoscope-Ch8\b"]:
- tool_name, tool_path, tool_pipe = find_tool_substitution(pattern)
- if not tool_path:
- # Provide a substitution anyway, for the sake of consistent errors.
- tool_path = config.llvm_tools_dir + '/' + tool_name
- config.substitutions.append((pattern, tool_pipe + tool_path))
+optional_tools = [
+ 'llvm-go', 'llvm-mt', 'Kaleidoscope-Ch3', 'Kaleidoscope-Ch4',
+ 'Kaleidoscope-Ch5', 'Kaleidoscope-Ch6', 'Kaleidoscope-Ch7',
+ 'Kaleidoscope-Ch8']
+llvm_config.add_tool_substitutions(optional_tools, config.llvm_tools_dir,
+ warn_missing=False)
### Targets
Modified: llvm/trunk/utils/lit/lit/llvm/__init__.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/llvm/__init__.py?rev=313928&r1=313927&r2=313928&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/llvm/__init__.py (original)
+++ llvm/trunk/utils/lit/lit/llvm/__init__.py Thu Sep 21 15:16:40 2017
@@ -1,9 +1,51 @@
-
from lit.llvm import config
+import lit.util
+import re
llvm_config = None
+class ToolFilter(object):
+ """
+ String-like class used to build regex substitution patterns for
+ llvm tools. Handles things like adding word-boundary patterns,
+ and filtering characters from the beginning an end of a tool name
+ """
+
+ def __init__(self, name, pre=None, post=None, verbatim=False):
+ """
+ Construct a ToolFilter.
+
+ name: the literal name of the substitution to look for.
+
+ pre: If specified, the substitution will not find matches where
+ the character immediately preceding the word-boundary that begins
+ `name` is any of the characters in the string `pre`.
+
+ post: If specified, the substitution will not find matches where
+ the character immediately after the word-boundary that ends `name`
+ is any of the characters specified in the string `post`.
+
+ verbatim: If True, `name` is an exact regex that is passed to the
+ underlying substitution
+ """
+ if verbatim:
+ self.regex = name
+ return
+
+ def not_in(chars, where=''):
+ if not chars:
+ return ''
+ pattern_str = '|'.join(re.escape(x) for x in chars)
+ return r'(?{}!({}))'.format(where, pattern_str)
+
+ self.regex = not_in(pre, '<') + r'\b' + name + r'\b' + not_in(post)
+
+ def __str__(self):
+ return self.regex
+
+
def initialize(lit_config, test_config):
global llvm_config
+
llvm_config = config.LLVMConfig(lit_config, test_config)
Modified: llvm/trunk/utils/lit/lit/llvm/config.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/llvm/config.py?rev=313928&r1=313927&r2=313928&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/llvm/config.py (original)
+++ llvm/trunk/utils/lit/lit/llvm/config.py Thu Sep 21 15:16:40 2017
@@ -136,22 +136,26 @@ class LLVMConfig(object):
if name in self.config.environment:
del self.config.environment[name]
- def feature_config(self, features, encoding = 'ascii'):
- # Ask llvm-config about the specified feature.
- arguments = [x for (x, _) in features]
+ def get_process_output(self, command):
try:
- config_path = os.path.join(self.config.llvm_tools_dir, 'llvm-config')
-
- llvm_config_cmd = subprocess.Popen(
- [config_path] + arguments,
- stdout = subprocess.PIPE,
- env=self.config.environment)
+ cmd = subprocess.Popen(
+ command, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, env=self.config.environment)
+ stdout, stderr = cmd.communicate()
+ stdout = lit.util.to_string(stdout)
+ stderr = lit.util.to_string(stderr)
+ return (stdout, stderr)
except OSError:
- self.lit_config.fatal("Could not find llvm-config in " + self.config.llvm_tools_dir)
+ self.lit_config.fatal("Could not run process %s" % command)
+
+ def feature_config(self, features):
+ # Ask llvm-config about the specified feature.
+ arguments = [x for (x, _) in features]
+ config_path = os.path.join(self.config.llvm_tools_dir, 'llvm-config')
- output, _ = llvm_config_cmd.communicate()
- output = output.decode(encoding)
+ output, _ = self.get_process_output([config_path] + arguments)
lines = output.split('\n')
+
for (feature_line, (_, patterns)) in zip(lines, features):
# We should have either a callable or a dictionary. If it's a
# dictionary, grep each key against the output and use the value if
@@ -163,3 +167,85 @@ class LLVMConfig(object):
for (re_pattern, feature) in patterns.items():
if re.search(re_pattern, feature_line):
self.config.available_features.add(feature)
+
+
+ # Note that when substituting %clang_cc1 also fill in the include directory of
+ # the builtin headers. Those are part of even a freestanding environment, but
+ # Clang relies on the driver to locate them.
+ def get_clang_builtin_include_dir(self, clang):
+ # FIXME: Rather than just getting the version, we should have clang print
+ # out its resource dir here in an easy to scrape form.
+ clang_dir, _ = self.get_process_output([clang, '-print-file-name=include'])
+
+ if not clang_dir:
+ self.lit_config.fatal("Couldn't find the include dir for Clang ('%s')" % clang)
+
+ clang_dir = clang_dir.strip()
+ if sys.platform in ['win32'] and not self.use_lit_shell:
+ # Don't pass dosish path separator to msys bash.exe.
+ clang_dir = clang_dir.replace('\\', '/')
+ # Ensure the result is an ascii string, across Python2.5+ - Python3.
+ return clang_dir
+
+ def make_itanium_abi_triple(self, triple):
+ m = re.match(r'(\w+)-(\w+)-(\w+)', triple)
+ if not m:
+ self.lit_config.fatal("Could not turn '%s' into Itanium ABI triple" % triple)
+ if m.group(3).lower() != 'win32':
+ # All non-win32 triples use the Itanium ABI.
+ return triple
+ return m.group(1) + '-' + m.group(2) + '-mingw32'
+
+ def make_msabi_triple(self, triple):
+ m = re.match(r'(\w+)-(\w+)-(\w+)', triple)
+ if not m:
+ self.lit_config.fatal("Could not turn '%s' into MS ABI triple" % triple)
+ isa = m.group(1).lower()
+ vendor = m.group(2).lower()
+ os = m.group(3).lower()
+ if os == 'win32':
+ # If the OS is win32, we're done.
+ return triple
+ if isa.startswith('x86') or isa == 'amd64' or re.match(r'i\d86', isa):
+ # For x86 ISAs, adjust the OS.
+ return isa + '-' + vendor + '-win32'
+ # -win32 is not supported for non-x86 targets; use a default.
+ return 'i686-pc-win32'
+
+ def add_tool_substitutions(self, tools, search_dirs, warn_missing = True):
+ if lit.util.is_string(search_dirs):
+ search_dirs = [search_dirs]
+
+ search_dirs = os.pathsep.join(search_dirs)
+ for tool in tools:
+ # Extract the tool name from the pattern. This relies on the tool
+ # name being surrounded by \b word match operators. If the
+ # pattern starts with "| ", include it in the string to be
+ # substituted.
+ if lit.util.is_string(tool):
+ tool = lit.util.make_word_regex(tool)
+ else:
+ tool = str(tool)
+
+ tool_match = re.match(r"^(\\)?((\| )?)\W+b([0-9A-Za-z-_\.]+)\\b\W*$",
+ tool)
+ if not tool_match:
+ continue
+
+ tool_pipe = tool_match.group(2)
+ tool_name = tool_match.group(4)
+ tool_path = lit.util.which(tool_name, search_dirs)
+ if not tool_path:
+ if warn_missing:
+ # Warn, but still provide a substitution.
+ self.lit_config.note('Did not find ' + tool_name + ' in %s' % search_dirs)
+ tool_path = self.config.llvm_tools_dir + '/' + tool_name
+
+ if tool_name == 'llc' and os.environ.get('LLVM_ENABLE_MACHINE_VERIFIER') == '1':
+ tool_path += ' -verify-machineinstrs'
+ if tool_name == 'llvm-go':
+ exe = getattr(self.config, 'go_executable', None)
+ if exe:
+ tool_path += " go=" + exe
+
+ self.config.substitutions.append((tool, tool_pipe + tool_path))
Modified: llvm/trunk/utils/lit/lit/util.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/util.py?rev=313928&r1=313927&r2=313928&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/util.py (original)
+++ llvm/trunk/utils/lit/lit/util.py Thu Sep 21 15:16:40 2017
@@ -36,6 +36,8 @@ def pythonize_bool(value):
return False
raise ValueError('"{}" is not a valid boolean'.format(value))
+def make_word_regex(word):
+ return r'\b' + word + r'\b'
def to_bytes(s):
"""Return the parameter as type 'bytes', possibly encoding it.
More information about the llvm-commits
mailing list