[lit] Refactor out some more common lit configuration code.

debuginfo-tests has need to reuse a lot of common configuration
from clang and lld, and in general it seems like all of the
projects which are tightly coupled (e.g. lld, clang, llvm, lldb,
etc) can benefit from knowing about one other.  For example,
lldb needs to know various things about how to run clang in its
test suite.  Since there's a lot of common substitutions and
operations that need to be shared among projects, sinking this
up into LLVM makes sense.

In addition, this patch introduces a function add_tool_substitution
which handles all the dirty intricacies of matching tool names
which was previously copied around the various config files.  This
is now a simple straightforward interface which is hard to mess
up.

Differential Revision: https://reviews.llvm.org/D37944

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@313919 91177308-0d34-0410-b5e6-96231b3b80d8

4 files changed, 171 insertions, 111 deletions

diff --git a/test/lit.cfg.py b/test/lit.cfg.py
index 87e2a06e387..28e770d6308 100644
--- a/test/lit.cfg.py
+++ b/test/lit.cfg.py
@@ -11,6 +11,7 @@ import subprocess
 import lit.util
 import lit.formats
 from lit.llvm import llvm_config
+from lit.llvm import ToolFilter
 
 # name: The name of this test suite.
 config.name = 'LLVM'
@@ -134,108 +135,39 @@ else:
 # The regex is a pre-assertion to avoid matching a preceding
 # dot, hyphen, carat, or slash (.foo, -foo, etc.).  Some patterns
 # also have a post-assertion to not match a trailing hyphen (foo-).
-NOJUNK = r"(?<!\.|-|\^|/|<)"
-
-
-def find_tool_substitution(pattern):
-    # Extract the tool name from the pattern.  This relies on the tool
-    # name being surrounded by \b word match operators.  If the
-    # pattern starts with "| ", include it in the string to be
-    # substituted.
-    tool_match = re.match(r"^(\\)?((\| )?)\W+b([0-9A-Za-z-_]+)\\b\W*$",
-                          pattern)
-    tool_pipe = tool_match.group(2)
-    tool_name = tool_match.group(4)
-    # Did the user specify the tool path + arguments? This allows things like
-    # llvm-lit "-Dllc=llc -enable-misched -verify-machineinstrs"
-    tool_path = lit_config.params.get(tool_name)
-    if tool_path is None:
-        tool_path = lit.util.which(tool_name, config.llvm_tools_dir)
-        if tool_path is None:
-            return tool_name, tool_path, tool_pipe
-    if (tool_name == "llc" and
-       'LLVM_ENABLE_MACHINE_VERIFIER' in os.environ and
-       os.environ['LLVM_ENABLE_MACHINE_VERIFIER'] == "1"):
-        tool_path += " -verify-machineinstrs"
-    if (tool_name == "llvm-go"):
-        tool_path += " go=" + config.go_executable
-    return tool_name, tool_path, tool_pipe
-
-
-for pattern in [r"\bbugpoint\b(?!-)",
-                NOJUNK + r"\bllc\b",
-                r"\blli\b",
-                r"\bllvm-ar\b",
-                r"\bllvm-as\b",
-                r"\bllvm-bcanalyzer\b",
-                r"\bllvm-config\b",
-                r"\bllvm-cov\b",
-                r"\bllvm-cxxdump\b",
-                r"\bllvm-cvtres\b",
-                r"\bllvm-diff\b",
-                r"\bllvm-dis\b",
-                r"\bllvm-dsymutil\b",
-                r"\bllvm-dwarfdump\b",
-                r"\bllvm-extract\b",
-                r"\bllvm-isel-fuzzer\b",
-                r"\bllvm-lib\b",
-                r"\bllvm-link\b",
-                r"\bllvm-lto\b",
-                r"\bllvm-lto2\b",
-                r"\bllvm-mc\b",
-                r"\bllvm-mcmarkup\b",
-                r"\bllvm-modextract\b",
-                r"\bllvm-nm\b",
-                r"\bllvm-objcopy\b",
-                r"\bllvm-objdump\b",
-                r"\bllvm-pdbutil\b",
-                r"\bllvm-profdata\b",
-                r"\bllvm-ranlib\b",
-                r"\bllvm-readobj\b",
-                r"\bllvm-rtdyld\b",
-                r"\bllvm-size\b",
-                r"\bllvm-split\b",
-                r"\bllvm-strings\b",
-                r"\bllvm-tblgen\b",
-                r"\bllvm-c-test\b",
-                r"\bllvm-cxxfilt\b",
-                r"\bllvm-xray\b",
-                NOJUNK + r"\bllvm-symbolizer\b",
-                NOJUNK + r"\bopt\b",
-                r"\bFileCheck\b",
-                r"\bobj2yaml\b",
-                NOJUNK + r"\bsancov\b",
-                NOJUNK + r"\bsanstats\b",
-                r"\byaml2obj\b",
-                r"\byaml-bench\b",
-                r"\bverify-uselistorder\b",
-                # Handle these specially as they are strings searched
-                # for during testing.
-                r"\| \bcount\b",
-                r"\| \bnot\b"]:
-    tool_name, tool_path, tool_pipe = find_tool_substitution(pattern)
-    if not tool_path:
-        # Warn, but still provide a substitution.
-        lit_config.note('Did not find ' + tool_name + ' in ' + config.llvm_tools_dir)
-        tool_path = config.llvm_tools_dir + '/' + tool_name
-    config.substitutions.append((pattern, tool_pipe + tool_path))
+JUNKCHARS = r".-^/<"
+
+required_tools = [
+    'lli', 'llvm-ar', 'llvm-as', 'llvm-bcanalyzer', 'llvm-config', 'llvm-cov',
+    'llvm-cxxdump', 'llvm-cvtres', 'llvm-diff', 'llvm-dis', 'llvm-dsymutil',
+    'llvm-dwarfdump', 'llvm-extract', 'llvm-isel-fuzzer', 'llvm-lib',
+    'llvm-link', 'llvm-lto', 'llvm-lto2', 'llvm-mc', 'llvm-mcmarkup',
+    'llvm-modextract', 'llvm-nm', 'llvm-objcopy', 'llvm-objdump',
+    'llvm-pdbutil', 'llvm-profdata', 'llvm-ranlib', 'llvm-readobj',
+    'llvm-rtdyld', 'llvm-size', 'llvm-split', 'llvm-strings', 'llvm-tblgen',
+    'llvm-c-test', 'llvm-cxxfilt', 'llvm-xray', 'yaml2obj', 'obj2yaml',
+    'FileCheck', 'yaml-bench', 'verify-uselistorder',
+    ToolFilter('bugpoint', post='-'),
+    ToolFilter('llc', pre=JUNKCHARS),
+    ToolFilter('llvm-symbolizer', pre=JUNKCHARS),
+    ToolFilter('opt', JUNKCHARS),
+    ToolFilter('sancov', pre=JUNKCHARS),
+    ToolFilter('sanstats', pre=JUNKCHARS),
+    # Handle these specially as they are strings searched for during testing.
+    ToolFilter(r'\| \bcount\b', verbatim=True),
+    ToolFilter(r'\| \bnot\b', verbatim=True)]
+
+llvm_config.add_tool_substitutions(required_tools, config.llvm_tools_dir)
 
 # For tools that are optional depending on the config, we won't warn
 # if they're missing.
-for pattern in [r"\bllvm-go\b",
-                r"\bllvm-mt\b",
-                r"\bKaleidoscope-Ch3\b",
-                r"\bKaleidoscope-Ch4\b",
-                r"\bKaleidoscope-Ch5\b",
-                r"\bKaleidoscope-Ch6\b",
-                r"\bKaleidoscope-Ch7\b",
-                r"\bKaleidoscope-Ch8\b"]:
-    tool_name, tool_path, tool_pipe = find_tool_substitution(pattern)
-    if not tool_path:
-        # Provide a substitution anyway, for the sake of consistent errors.
-        tool_path = config.llvm_tools_dir + '/' + tool_name
-    config.substitutions.append((pattern, tool_pipe + tool_path))
 
+optional_tools = [
+    'llvm-go', 'llvm-mt', 'Kaleidoscope-Ch3', 'Kaleidoscope-Ch4',
+    'Kaleidoscope-Ch5', 'Kaleidoscope-Ch6', 'Kaleidoscope-Ch7',
+    'Kaleidoscope-Ch8']
+llvm_config.add_tool_substitutions(optional_tools, config.llvm_tools_dir,
+                                   warn_missing=False)
 
 ### Targets
 
diff --git a/utils/lit/lit/llvm/__init__.py b/utils/lit/lit/llvm/__init__.py
index c4cad04649f..4a9249978dd 100644
--- a/utils/lit/lit/llvm/__init__.py
+++ b/utils/lit/lit/llvm/__init__.py
@@ -1,9 +1,51 @@
-
 from lit.llvm import config
+import lit.util
+import re
 
 llvm_config = None
 
+class ToolFilter(object):
+    """
+        String-like class used to build regex substitution patterns for
+        llvm tools.  Handles things like adding word-boundary patterns,
+        and filtering characters from the beginning an end of a tool name
+    """
+
+    def __init__(self, name, pre=None, post=None, verbatim=False):
+        """
+            Construct a ToolFilter.
+
+            name: the literal name of the substitution to look for.
+
+            pre: If specified, the substitution will not find matches where
+            the character immediately preceding the word-boundary that begins
+            `name` is any of the characters in the string `pre`.
+
+            post: If specified, the substitution will not find matches where
+            the character immediately after the word-boundary that ends `name`
+            is any of the characters specified in the string `post`.
+
+            verbatim: If True, `name` is an exact regex that is passed to the
+            underlying substitution
+        """
+        if verbatim:
+            self.regex = name
+            return
+
+        def not_in(chars, where=''):
+            if not chars:
+                return ''
+            pattern_str = '|'.join(re.escape(x) for x in chars)
+            return r'(?{}!({}))'.format(where, pattern_str)
+
+        self.regex = not_in(pre, '<') + r'\b' + name + r'\b' + not_in(post)
+
+    def __str__(self):
+        return self.regex
+
+
 def initialize(lit_config, test_config):
     global llvm_config
+
     llvm_config = config.LLVMConfig(lit_config, test_config)
 
diff --git a/utils/lit/lit/llvm/config.py b/utils/lit/lit/llvm/config.py
index d2db46135ad..cc0f6750444 100644
--- a/utils/lit/lit/llvm/config.py
+++ b/utils/lit/lit/llvm/config.py
@@ -136,22 +136,24 @@ class LLVMConfig(object):
             if name in self.config.environment:
                 del self.config.environment[name]
 
-    def feature_config(self, features, encoding = 'ascii'):
-        # Ask llvm-config about the specified feature.
-        arguments = [x for (x, _) in features]
+    def get_process_output(self, command):
         try:
-            config_path = os.path.join(self.config.llvm_tools_dir, 'llvm-config')
-
-            llvm_config_cmd = subprocess.Popen(
-                [config_path] + arguments,
-                stdout = subprocess.PIPE,
-                env=self.config.environment)
+            cmd = subprocess.Popen(
+                command, stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE, env=self.config.environment)
+            stdout, stderr = cmd.communicate()
+            return (stdout, stderr)
         except OSError:
-            self.lit_config.fatal("Could not find llvm-config in " + self.config.llvm_tools_dir)
+            self.lit_config.fatal("Could not run process %s" % command)
+
+    def feature_config(self, features):
+        # Ask llvm-config about the specified feature.
+        arguments = [x for (x, _) in features]
+        config_path = os.path.join(self.config.llvm_tools_dir, 'llvm-config')
 
-        output, _ = llvm_config_cmd.communicate()
-        output = output.decode(encoding)
+        output, _ = self.get_process_output([config_path] + arguments)
         lines = output.split('\n')
+
         for (feature_line, (_, patterns)) in zip(lines, features):
             # We should have either a callable or a dictionary.  If it's a
             # dictionary, grep each key against the output and use the value if
@@ -163,3 +165,85 @@ class LLVMConfig(object):
                 for (re_pattern, feature) in patterns.items():
                     if re.search(re_pattern, feature_line):
                         self.config.available_features.add(feature)
+
+
+    # Note that when substituting %clang_cc1 also fill in the include directory of
+    # the builtin headers. Those are part of even a freestanding environment, but
+    # Clang relies on the driver to locate them.
+    def get_clang_builtin_include_dir(self, clang):
+        # FIXME: Rather than just getting the version, we should have clang print
+        # out its resource dir here in an easy to scrape form.
+        clang_dir, _ = self.get_process_output([clang, '-print-file-name=include'])
+
+        if not clang_dir:
+          self.lit_config.fatal("Couldn't find the include dir for Clang ('%s')" % clang)
+
+        clang_dir = clang_dir.strip()
+        if sys.platform in ['win32'] and not self.use_lit_shell:
+            # Don't pass dosish path separator to msys bash.exe.
+            clang_dir = clang_dir.replace('\\', '/')
+        # Ensure the result is an ascii string, across Python2.5+ - Python3.
+        return clang_dir
+
+    def make_itanium_abi_triple(self, triple):
+        m = re.match(r'(\w+)-(\w+)-(\w+)', triple)
+        if not m:
+          self.lit_config.fatal("Could not turn '%s' into Itanium ABI triple" % triple)
+        if m.group(3).lower() != 'win32':
+          # All non-win32 triples use the Itanium ABI.
+          return triple
+        return m.group(1) + '-' + m.group(2) + '-mingw32'
+
+    def make_msabi_triple(self, triple):
+        m = re.match(r'(\w+)-(\w+)-(\w+)', triple)
+        if not m:
+          self.lit_config.fatal("Could not turn '%s' into MS ABI triple" % triple)
+        isa = m.group(1).lower()
+        vendor = m.group(2).lower()
+        os = m.group(3).lower()
+        if os == 'win32':
+          # If the OS is win32, we're done.
+          return triple
+        if isa.startswith('x86') or isa == 'amd64' or re.match(r'i\d86', isa):
+          # For x86 ISAs, adjust the OS.
+          return isa + '-' + vendor + '-win32'
+        # -win32 is not supported for non-x86 targets; use a default.
+        return 'i686-pc-win32'
+
+    def add_tool_substitutions(self, tools, search_dirs, warn_missing = True):
+        if lit.util.is_string(search_dirs):
+            search_dirs = [search_dirs]
+
+        search_dirs = os.pathsep.join(search_dirs)
+        for tool in tools:
+            # Extract the tool name from the pattern.  This relies on the tool
+            # name being surrounded by \b word match operators.  If the
+            # pattern starts with "| ", include it in the string to be
+            # substituted.
+            if lit.util.is_string(tool):
+                tool = lit.util.make_word_regex(tool)
+            else:
+                tool = str(tool)
+
+            tool_match = re.match(r"^(\\)?((\| )?)\W+b([0-9A-Za-z-_\.]+)\\b\W*$",
+                                  tool)
+            if not tool_match:
+                continue
+
+            tool_pipe = tool_match.group(2)
+            tool_name = tool_match.group(4)
+            tool_path = lit.util.which(tool_name, search_dirs)
+            if not tool_path:
+                if warn_missing:
+                    # Warn, but still provide a substitution.
+                    self.lit_config.note('Did not find ' + tool_name + ' in %s' % search_dirs)
+                tool_path = self.config.llvm_tools_dir + '/' + tool_name
+
+            if tool_name == 'llc' and os.environ.get('LLVM_ENABLE_MACHINE_VERIFIER') == '1':
+                tool_path += ' -verify-machineinstrs'
+            if tool_name == 'llvm-go':
+                exe = getattr(self.config, 'go_executable', None)
+                if exe:
+                    tool_path += " go=" + exe
+
+            self.config.substitutions.append((tool, tool_pipe + tool_path))
diff --git a/utils/lit/lit/util.py b/utils/lit/lit/util.py
index 174194f81be..e072a9ef81b 100644
--- a/utils/lit/lit/util.py
+++ b/utils/lit/lit/util.py
@@ -36,6 +36,8 @@ def pythonize_bool(value):
             return False
     raise ValueError('"{}" is not a valid boolean'.format(value))
 
+def make_word_regex(word):
+    return r'\b' + word + r'\b'
 
 def to_bytes(s):
     """Return the parameter as type 'bytes', possibly encoding it.