Resubmit "[lit] Refactor out some more common lit configuration code."

There were two issues, one Python 3 specific related to Unicode,
and another which is that the tool substitution for lld no longer
rejected matches where a / preceded the tool name.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@313928 91177308-0d34-0410-b5e6-96231b3b80d8

3 files changed, 143 insertions, 13 deletions

diff --git a/utils/lit/lit/llvm/__init__.py b/utils/lit/lit/llvm/__init__.py
index c4cad04649f..4a9249978dd 100644
--- a/utils/lit/lit/llvm/__init__.py
+++ b/utils/lit/lit/llvm/__init__.py
@@ -1,9 +1,51 @@
-
 from lit.llvm import config
+import lit.util
+import re
 
 llvm_config = None
 
+class ToolFilter(object):
+    """
+        String-like class used to build regex substitution patterns for
+        llvm tools.  Handles things like adding word-boundary patterns,
+        and filtering characters from the beginning an end of a tool name
+    """
+
+    def __init__(self, name, pre=None, post=None, verbatim=False):
+        """
+            Construct a ToolFilter.
+
+            name: the literal name of the substitution to look for.
+
+            pre: If specified, the substitution will not find matches where
+            the character immediately preceding the word-boundary that begins
+            `name` is any of the characters in the string `pre`.
+
+            post: If specified, the substitution will not find matches where
+            the character immediately after the word-boundary that ends `name`
+            is any of the characters specified in the string `post`.
+
+            verbatim: If True, `name` is an exact regex that is passed to the
+            underlying substitution
+        """
+        if verbatim:
+            self.regex = name
+            return
+
+        def not_in(chars, where=''):
+            if not chars:
+                return ''
+            pattern_str = '|'.join(re.escape(x) for x in chars)
+            return r'(?{}!({}))'.format(where, pattern_str)
+
+        self.regex = not_in(pre, '<') + r'\b' + name + r'\b' + not_in(post)
+
+    def __str__(self):
+        return self.regex
+
+
 def initialize(lit_config, test_config):
     global llvm_config
+
     llvm_config = config.LLVMConfig(lit_config, test_config)
 
diff --git a/utils/lit/lit/llvm/config.py b/utils/lit/lit/llvm/config.py
index d2db46135ad..174279c31e4 100644
--- a/utils/lit/lit/llvm/config.py
+++ b/utils/lit/lit/llvm/config.py
@@ -136,22 +136,26 @@ class LLVMConfig(object):
             if name in self.config.environment:
                 del self.config.environment[name]
 
-    def feature_config(self, features, encoding = 'ascii'):
-        # Ask llvm-config about the specified feature.
-        arguments = [x for (x, _) in features]
+    def get_process_output(self, command):
         try:
-            config_path = os.path.join(self.config.llvm_tools_dir, 'llvm-config')
-
-            llvm_config_cmd = subprocess.Popen(
-                [config_path] + arguments,
-                stdout = subprocess.PIPE,
-                env=self.config.environment)
+            cmd = subprocess.Popen(
+                command, stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE, env=self.config.environment)
+            stdout, stderr = cmd.communicate()
+            stdout = lit.util.to_string(stdout)
+            stderr = lit.util.to_string(stderr)
+            return (stdout, stderr)
         except OSError:
-            self.lit_config.fatal("Could not find llvm-config in " + self.config.llvm_tools_dir)
+            self.lit_config.fatal("Could not run process %s" % command)
+
+    def feature_config(self, features):
+        # Ask llvm-config about the specified feature.
+        arguments = [x for (x, _) in features]
+        config_path = os.path.join(self.config.llvm_tools_dir, 'llvm-config')
 
-        output, _ = llvm_config_cmd.communicate()
-        output = output.decode(encoding)
+        output, _ = self.get_process_output([config_path] + arguments)
         lines = output.split('\n')
+
         for (feature_line, (_, patterns)) in zip(lines, features):
             # We should have either a callable or a dictionary.  If it's a
             # dictionary, grep each key against the output and use the value if
@@ -163,3 +167,85 @@ class LLVMConfig(object):
                 for (re_pattern, feature) in patterns.items():
                     if re.search(re_pattern, feature_line):
                         self.config.available_features.add(feature)
+
+
+    # Note that when substituting %clang_cc1 also fill in the include directory of
+    # the builtin headers. Those are part of even a freestanding environment, but
+    # Clang relies on the driver to locate them.
+    def get_clang_builtin_include_dir(self, clang):
+        # FIXME: Rather than just getting the version, we should have clang print
+        # out its resource dir here in an easy to scrape form.
+        clang_dir, _ = self.get_process_output([clang, '-print-file-name=include'])
+
+        if not clang_dir:
+          self.lit_config.fatal("Couldn't find the include dir for Clang ('%s')" % clang)
+
+        clang_dir = clang_dir.strip()
+        if sys.platform in ['win32'] and not self.use_lit_shell:
+            # Don't pass dosish path separator to msys bash.exe.
+            clang_dir = clang_dir.replace('\\', '/')
+        # Ensure the result is an ascii string, across Python2.5+ - Python3.
+        return clang_dir
+
+    def make_itanium_abi_triple(self, triple):
+        m = re.match(r'(\w+)-(\w+)-(\w+)', triple)
+        if not m:
+          self.lit_config.fatal("Could not turn '%s' into Itanium ABI triple" % triple)
+        if m.group(3).lower() != 'win32':
+          # All non-win32 triples use the Itanium ABI.
+          return triple
+        return m.group(1) + '-' + m.group(2) + '-mingw32'
+
+    def make_msabi_triple(self, triple):
+        m = re.match(r'(\w+)-(\w+)-(\w+)', triple)
+        if not m:
+          self.lit_config.fatal("Could not turn '%s' into MS ABI triple" % triple)
+        isa = m.group(1).lower()
+        vendor = m.group(2).lower()
+        os = m.group(3).lower()
+        if os == 'win32':
+          # If the OS is win32, we're done.
+          return triple
+        if isa.startswith('x86') or isa == 'amd64' or re.match(r'i\d86', isa):
+          # For x86 ISAs, adjust the OS.
+          return isa + '-' + vendor + '-win32'
+        # -win32 is not supported for non-x86 targets; use a default.
+        return 'i686-pc-win32'
+
+    def add_tool_substitutions(self, tools, search_dirs, warn_missing = True):
+        if lit.util.is_string(search_dirs):
+            search_dirs = [search_dirs]
+
+        search_dirs = os.pathsep.join(search_dirs)
+        for tool in tools:
+            # Extract the tool name from the pattern.  This relies on the tool
+            # name being surrounded by \b word match operators.  If the
+            # pattern starts with "| ", include it in the string to be
+            # substituted.
+            if lit.util.is_string(tool):
+                tool = lit.util.make_word_regex(tool)
+            else:
+                tool = str(tool)
+
+            tool_match = re.match(r"^(\\)?((\| )?)\W+b([0-9A-Za-z-_\.]+)\\b\W*$",
+                                  tool)
+            if not tool_match:
+                continue
+
+            tool_pipe = tool_match.group(2)
+            tool_name = tool_match.group(4)
+            tool_path = lit.util.which(tool_name, search_dirs)
+            if not tool_path:
+                if warn_missing:
+                    # Warn, but still provide a substitution.
+                    self.lit_config.note('Did not find ' + tool_name + ' in %s' % search_dirs)
+                tool_path = self.config.llvm_tools_dir + '/' + tool_name
+
+            if tool_name == 'llc' and os.environ.get('LLVM_ENABLE_MACHINE_VERIFIER') == '1':
+                tool_path += ' -verify-machineinstrs'
+            if tool_name == 'llvm-go':
+                exe = getattr(self.config, 'go_executable', None)
+                if exe:
+                    tool_path += " go=" + exe
+
+            self.config.substitutions.append((tool, tool_pipe + tool_path))
diff --git a/utils/lit/lit/util.py b/utils/lit/lit/util.py
index 174194f81be..e072a9ef81b 100644
--- a/utils/lit/lit/util.py
+++ b/utils/lit/lit/util.py
@@ -36,6 +36,8 @@ def pythonize_bool(value):
             return False
     raise ValueError('"{}" is not a valid boolean'.format(value))
 
+def make_word_regex(word):
+    return r'\b' + word + r'\b'
 
 def to_bytes(s):
     """Return the parameter as type 'bytes', possibly encoding it.