diff options
| -rw-r--r-- | docs/TestingGuide.rst | 64 | ||||
| -rw-r--r-- | utils/lit/lit/BooleanExpression.py | 251 | ||||
| -rw-r--r-- | utils/lit/lit/Test.py | 118 | ||||
| -rw-r--r-- | utils/lit/lit/TestRunner.py | 165 | ||||
| -rw-r--r-- | utils/lit/tests/Inputs/shtest-format/requires-missing.txt | 7 | ||||
| -rw-r--r-- | utils/lit/tests/Inputs/shtest-format/requires-present.txt | 4 | ||||
| -rw-r--r-- | utils/lit/tests/Inputs/shtest-format/requires-star.txt | 3 | ||||
| -rw-r--r-- | utils/lit/tests/Inputs/shtest-format/requires-triple.txt | 3 | ||||
| -rw-r--r-- | utils/lit/tests/Inputs/shtest-format/unsupported-expr-false.txt | 9 | ||||
| -rw-r--r-- | utils/lit/tests/Inputs/shtest-format/unsupported-expr-true.txt | 4 | ||||
| -rw-r--r-- | utils/lit/tests/Inputs/shtest-format/unsupported-star.txt | 3 | ||||
| -rw-r--r-- | utils/lit/tests/Inputs/shtest-format/xfail-expr-false.txt | 3 | ||||
| -rw-r--r-- | utils/lit/tests/Inputs/shtest-format/xfail-expr-true.txt | 4 | ||||
| -rw-r--r-- | utils/lit/tests/boolean-parsing.py | 4 | ||||
| -rw-r--r-- | utils/lit/tests/shtest-format.py | 15 | ||||
| -rw-r--r-- | utils/lit/tests/unit/TestRunner.py | 57 |
16 files changed, 112 insertions, 602 deletions
diff --git a/docs/TestingGuide.rst b/docs/TestingGuide.rst index 99616770d8e..26143febd07 100644 --- a/docs/TestingGuide.rst +++ b/docs/TestingGuide.rst @@ -387,49 +387,23 @@ depends on special features of sub-architectures, you must add the specific triple, test with the specific FileCheck and put it into the specific directory that will filter out all other architectures. +REQUIRES and REQUIRES-ANY directive +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Constraining test execution ---------------------------- - -Some tests can be run only in specific configurations, such as -with debug builds or on particular platforms. Use ``REQUIRES`` -and ``UNSUPPORTED`` to control when the test is enabled. - -Some tests are expected to fail. For example, there may be a known bug -that the test detect. Use ``XFAIL`` to mark a test as an expected failure. -An ``XFAIL`` test will be successful if its execution fails, and -will be a failure if its execution succeeds. +Some tests can be enabled only in specific situation - like having +debug build. Use ``REQUIRES`` directive to specify those requirements. .. code-block:: llvm - ; This test will be only enabled in the build with asserts. + ; This test will be only enabled in the build with asserts ; REQUIRES: asserts - ; This test is disabled on Linux. - ; UNSUPPORTED: -linux- - ; This test is expected to fail on PowerPC. - ; XFAIL: powerpc - -``REQUIRES`` and ``UNSUPPORTED`` and ``XFAIL`` all accept a comma-separated -list of boolean expressions. The values in each expression may be: -- Features added to ``config.available_features`` by - configuration files such as ``lit.cfg``. -- Substrings of the target triple (``UNSUPPORTED`` and ``XFAIL`` only). - -| ``REQUIRES`` enables the test if all expressions are true. -| ``UNSUPPORTED`` disables the test if any expression is true. -| ``XFAIL`` expects the test to fail if any expression is true. - -As a special case, ``XFAIL: *`` is expected to fail everywhere. - -.. code-block:: llvm - - ; This test is disabled on Windows, - ; and is disabled on Linux, except for Android Linux. - ; UNSUPPORTED: windows, linux && !android - ; This test is expected to fail on both PowerPC and ARM. - ; XFAIL: powerpc || arm +You can separate requirements by a comma. +``REQUIRES`` means all listed requirements must be satisfied. +``REQUIRES-ANY`` means at least one must be satisfied. +List of features that can be used in ``REQUIRES`` and ``REQUIRES-ANY`` can be +found in lit.cfg files. Substitutions ------------- @@ -546,6 +520,24 @@ their name. For example: This program runs its arguments and then inverts the result code from it. Zero result codes become 1. Non-zero result codes become 0. +Sometimes it is necessary to mark a test case as "expected fail" or +XFAIL. You can easily mark a test as XFAIL just by including ``XFAIL:`` +on a line near the top of the file. This signals that the test case +should succeed if the test fails. Such test cases are counted separately +by the testing tool. To specify an expected fail, use the XFAIL keyword +in the comments of the test program followed by a colon and one or more +failure patterns. Each failure pattern can be either ``*`` (to specify +fail everywhere), or a part of a target triple (indicating the test +should fail on that platform), or the name of a configurable feature +(for example, ``loadable_module``). If there is a match, the test is +expected to fail. If not, the test is expected to succeed. To XFAIL +everywhere just specify ``XFAIL: *``. Here is an example of an ``XFAIL`` +line: + +.. code-block:: llvm + + ; XFAIL: darwin,sun + To make the output more useful, :program:`lit` will scan the lines of the test case for ones that contain a pattern that matches ``PR[0-9]+``. This is the syntax for specifying a PR (Problem Report) number diff --git a/utils/lit/lit/BooleanExpression.py b/utils/lit/lit/BooleanExpression.py deleted file mode 100644 index 3eb5060de3e..00000000000 --- a/utils/lit/lit/BooleanExpression.py +++ /dev/null @@ -1,251 +0,0 @@ -import re - -class BooleanExpression: - # A simple evaluator of boolean expressions. - # - # Grammar: - # expr :: or_expr - # or_expr :: and_expr ('||' and_expr)* - # and_expr :: not_expr ('&&' not_expr)* - # not_expr :: '!' not_expr - # '(' or_expr ')' - # identifier - # identifier :: [-+=._a-zA-Z0-9]+ - - # Evaluates `string` as a boolean expression. - # Returns True or False. Throws a ValueError on syntax error. - # - # Variables in `variables` are true. - # Substrings of `triple` are true. - # 'true' is true. - # All other identifiers are false. - @staticmethod - def evaluate(string, variables, triple=""): - try: - parser = BooleanExpression(string, set(variables), triple) - return parser.parseAll() - except ValueError as e: - raise ValueError(str(e) + ('\nin expression: %r' % string)) - - ##### - - def __init__(self, string, variables, triple=""): - self.tokens = BooleanExpression.tokenize(string) - self.variables = variables - self.variables.add('true') - self.triple = triple - self.value = None - self.token = None - - # Singleton end-of-expression marker. - END = object() - - # Tokenization pattern. - Pattern = re.compile(r'\A\s*([()]|[-+=._a-zA-Z0-9]+|&&|\|\||!)\s*(.*)\Z') - - @staticmethod - def tokenize(string): - while True: - m = re.match(BooleanExpression.Pattern, string) - if m is None: - if string == "": - yield BooleanExpression.END; - return - else: - raise ValueError("couldn't parse text: %r" % string) - - token = m.group(1) - string = m.group(2) - yield token - - def quote(self, token): - if token is BooleanExpression.END: - return '<end of expression>' - else: - return repr(token) - - def accept(self, t): - if self.token == t: - self.token = next(self.tokens) - return True - else: - return False - - def expect(self, t): - if self.token == t: - if self.token != BooleanExpression.END: - self.token = next(self.tokens) - else: - raise ValueError("expected: %s\nhave: %s" % - (self.quote(t), self.quote(self.token))) - - def isIdentifier(self, t): - if (t is BooleanExpression.END or t == '&&' or t == '||' or - t == '!' or t == '(' or t == ')'): - return False - return True - - def parseNOT(self): - if self.accept('!'): - self.parseNOT() - self.value = not self.value - elif self.accept('('): - self.parseOR() - self.expect(')') - elif not self.isIdentifier(self.token): - raise ValueError("expected: '!' or '(' or identifier\nhave: %s" % - self.quote(self.token)) - else: - self.value = (self.token in self.variables or - self.token in self.triple) - self.token = next(self.tokens) - - def parseAND(self): - self.parseNOT() - while self.accept('&&'): - left = self.value - self.parseNOT() - right = self.value - # this is technically the wrong associativity, but it - # doesn't matter for this limited expression grammar - self.value = left and right - - def parseOR(self): - self.parseAND() - while self.accept('||'): - left = self.value - self.parseAND() - right = self.value - # this is technically the wrong associativity, but it - # doesn't matter for this limited expression grammar - self.value = left or right - - def parseAll(self): - self.token = next(self.tokens) - self.parseOR() - self.expect(BooleanExpression.END) - return self.value - - -####### -# Tests - -import unittest - -class TestBooleanExpression(unittest.TestCase): - def test_variables(self): - variables = {'its-true', 'false-lol-true', 'under_score', - 'e=quals', 'd1g1ts'} - self.assertTrue(BooleanExpression.evaluate('true', variables)) - self.assertTrue(BooleanExpression.evaluate('its-true', variables)) - self.assertTrue(BooleanExpression.evaluate('false-lol-true', variables)) - self.assertTrue(BooleanExpression.evaluate('under_score', variables)) - self.assertTrue(BooleanExpression.evaluate('e=quals', variables)) - self.assertTrue(BooleanExpression.evaluate('d1g1ts', variables)) - - self.assertFalse(BooleanExpression.evaluate('false', variables)) - self.assertFalse(BooleanExpression.evaluate('True', variables)) - self.assertFalse(BooleanExpression.evaluate('true-ish', variables)) - self.assertFalse(BooleanExpression.evaluate('not_true', variables)) - self.assertFalse(BooleanExpression.evaluate('tru', variables)) - - def test_triple(self): - triple = 'arch-vendor-os' - self.assertTrue(BooleanExpression.evaluate('arch-', {}, triple)) - self.assertTrue(BooleanExpression.evaluate('ar', {}, triple)) - self.assertTrue(BooleanExpression.evaluate('ch-vend', {}, triple)) - self.assertTrue(BooleanExpression.evaluate('-vendor-', {}, triple)) - self.assertTrue(BooleanExpression.evaluate('-os', {}, triple)) - self.assertFalse(BooleanExpression.evaluate('arch-os', {}, triple)) - - def test_operators(self): - self.assertTrue(BooleanExpression.evaluate('true || true', {})) - self.assertTrue(BooleanExpression.evaluate('true || false', {})) - self.assertTrue(BooleanExpression.evaluate('false || true', {})) - self.assertFalse(BooleanExpression.evaluate('false || false', {})) - - self.assertTrue(BooleanExpression.evaluate('true && true', {})) - self.assertFalse(BooleanExpression.evaluate('true && false', {})) - self.assertFalse(BooleanExpression.evaluate('false && true', {})) - self.assertFalse(BooleanExpression.evaluate('false && false', {})) - - self.assertFalse(BooleanExpression.evaluate('!true', {})) - self.assertTrue(BooleanExpression.evaluate('!false', {})) - - self.assertTrue(BooleanExpression.evaluate(' ((!((false) )) ) ', {})) - self.assertTrue(BooleanExpression.evaluate('true && (true && (true))', {})) - self.assertTrue(BooleanExpression.evaluate('!false && !false && !! !false', {})) - self.assertTrue(BooleanExpression.evaluate('false && false || true', {})) - self.assertTrue(BooleanExpression.evaluate('(false && false) || true', {})) - self.assertFalse(BooleanExpression.evaluate('false && (false || true)', {})) - - # Evaluate boolean expression `expr`. - # Fail if it does not throw a ValueError containing the text `error`. - def checkException(self, expr, error): - try: - BooleanExpression.evaluate(expr, {}) - self.fail("expression %r didn't cause an exception" % expr) - except ValueError as e: - if -1 == str(e).find(error): - self.fail(("expression %r caused the wrong ValueError\n" + - "actual error was:\n%s\n" + - "expected error was:\n%s\n") % (expr, e, error)) - except BaseException as e: - self.fail(("expression %r caused the wrong exception; actual " + - "exception was: \n%r") % (expr, e)) - - def test_errors(self): - self.checkException("ba#d", - "couldn't parse text: '#d'\n" + - "in expression: 'ba#d'") - - self.checkException("true and true", - "expected: <end of expression>\n" + - "have: 'and'\n" + - "in expression: 'true and true'") - - self.checkException("|| true", - "expected: '!' or '(' or identifier\n" + - "have: '||'\n" + - "in expression: '|| true'") - - self.checkException("true &&", - "expected: '!' or '(' or identifier\n" + - "have: <end of expression>\n" + - "in expression: 'true &&'") - - self.checkException("", - "expected: '!' or '(' or identifier\n" + - "have: <end of expression>\n" + - "in expression: ''") - - self.checkException("*", - "couldn't parse text: '*'\n" + - "in expression: '*'") - - self.checkException("no wait stop", - "expected: <end of expression>\n" + - "have: 'wait'\n" + - "in expression: 'no wait stop'") - - self.checkException("no-$-please", - "couldn't parse text: '$-please'\n" + - "in expression: 'no-$-please'") - - self.checkException("(((true && true) || true)", - "expected: ')'\n" + - "have: <end of expression>\n" + - "in expression: '(((true && true) || true)'") - - self.checkException("true (true)", - "expected: <end of expression>\n" + - "have: '('\n" + - "in expression: 'true (true)'") - - self.checkException("( )", - "expected: '!' or '(' or identifier\n" + - "have: ')'\n" + - "in expression: '( )'") - -if __name__ == '__main__': - unittest.main() diff --git a/utils/lit/lit/Test.py b/utils/lit/lit/Test.py index 1a9e3fe80fb..657a7e8140d 100644 --- a/utils/lit/lit/Test.py +++ b/utils/lit/lit/Test.py @@ -2,8 +2,6 @@ import os from xml.sax.saxutils import escape from json import JSONEncoder -from lit.BooleanExpression import BooleanExpression - # Test result codes. class ResultCode(object): @@ -182,24 +180,10 @@ class Test: self.path_in_suite = path_in_suite self.config = config self.file_path = file_path - - # A list of conditions under which this test is expected to fail. - # Each condition is a boolean expression of features and target - # triple parts. These can optionally be provided by test format - # handlers, and will be honored when the test result is supplied. + # A list of conditions under which this test is expected to fail. These + # can optionally be provided by test format handlers, and will be + # honored when the test result is supplied. self.xfails = [] - - # A list of conditions that must be satisfied before running the test. - # Each condition is a boolean expression of features. All of them - # must be True for the test to run. - # FIXME should target triple parts count here too? - self.requires = [] - - # A list of conditions that prevent execution of the test. - # Each condition is a boolean expression of features and target - # triple parts. All of them must be False for the test to run. - self.unsupported = [] - # The test result, once complete. self.result = None @@ -212,16 +196,11 @@ class Test: self.result = result # Apply the XFAIL handling to resolve the result exit code. - try: - if self.isExpectedToFail(): - if self.result.code == PASS: - self.result.code = XPASS - elif self.result.code == FAIL: - self.result.code = XFAIL - except ValueError as e: - # Syntax error in an XFAIL line. - self.result.code = UNRESOLVED - self.result.output = str(e) + if self.isExpectedToFail(): + if self.result.code == PASS: + self.result.code = XPASS + elif self.result.code == FAIL: + self.result.code = XFAIL def getFullName(self): return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite) @@ -245,90 +224,23 @@ class Test: configuration. This check relies on the test xfails property which by some test formats may not be computed until the test has first been executed. - Throws ValueError if an XFAIL line has a syntax error. """ - features = self.config.available_features - triple = getattr(self.suite.config, 'target_triple', "") - # Check if any of the xfails match an available feature or the target. for item in self.xfails: # If this is the wildcard, it always fails. if item == '*': return True - # If this is a True expression of features and target triple parts, - # it fails. - try: - if BooleanExpression.evaluate(item, features, triple): - return True - except ValueError as e: - raise ValueError('Error in XFAIL list:\n%s' % str(e)) - - return False - - def isWithinFeatureLimits(self): - """ - isWithinFeatureLimits() -> bool - - A test is within the feature limits set by run_only_tests if - 1. the test's requirements ARE satisfied by the available features - 2. the test's requirements ARE NOT satisfied after the limiting - features are removed from the available features - - Throws ValueError if a REQUIRES line has a syntax error. - """ - - if not self.config.limit_to_features: - return True # No limits. Run it. - - # Check the requirements as-is (#1) - if self.getMissingRequiredFeatures(): - return False - - # Check the requirements after removing the limiting features (#2) - featuresMinusLimits = [f for f in self.config.available_features - if not f in self.config.limit_to_features] - if not self.getMissingRequiredFeaturesFromList(featuresMinusLimits): - return False - - return True - - def getMissingRequiredFeaturesFromList(self, features): - try: - return [item for item in self.requires - if not BooleanExpression.evaluate(item, features)] - except ValueError as e: - raise ValueError('Error in REQUIRES list:\n%s' % str(e)) - - def getMissingRequiredFeatures(self): - """ - getMissingRequiredFeatures() -> list of strings - - Returns a list of features from REQUIRES that are not satisfied." - Throws ValueError if a REQUIRES line has a syntax error. - """ - - features = self.config.available_features - return self.getMissingRequiredFeaturesFromList(features) - - def getUnsupportedFeatures(self): - """ - getUnsupportedFeatures() -> list of strings - - Returns a list of features from UNSUPPORTED that are present - in the test configuration's features or target triple. - Throws ValueError if an UNSUPPORTED line has a syntax error. - """ + # If this is an exact match for one of the features, it fails. + if item in self.config.available_features: + return True - features = self.config.available_features - triple = getattr(self.suite.config, 'target_triple', "") + # If this is a part of the target triple, it fails. + if item and item in self.suite.config.target_triple: + return True - try: - return [item for item in self.unsupported - if BooleanExpression.evaluate(item, features, triple)] - except ValueError as e: - raise ValueError('Error in UNSUPPORTED list:\n%s' % str(e)) + return False def isEarlyTest(self): """ diff --git a/utils/lit/lit/TestRunner.py b/utils/lit/lit/TestRunner.py index 9f9ff199f9a..942ae38a04c 100644 --- a/utils/lit/lit/TestRunner.py +++ b/utils/lit/lit/TestRunner.py @@ -9,7 +9,6 @@ import lit.ShUtil as ShUtil import lit.Test as Test import lit.util from lit.util import to_bytes, to_string -from lit.BooleanExpression import BooleanExpression class InternalShellError(Exception): def __init__(self, command, message): @@ -747,35 +746,14 @@ class ParserKind(object): command. TAG: A keyword taking no value. Ex 'END.' - COMMAND: A keyword taking a list of shell commands. Ex 'RUN:' - LIST: A keyword taking a comma-separated list of values. - BOOLEAN_EXPR: A keyword taking a comma-separated list of - boolean expressions. Ex 'XFAIL:' + COMMAND: A Keyword taking a list of shell commands. Ex 'RUN:' + LIST: A keyword taking a comma separated list of value. Ex 'XFAIL:' CUSTOM: A keyword with custom parsing semantics. """ TAG = 0 COMMAND = 1 LIST = 2 - BOOLEAN_EXPR = 3 - CUSTOM = 4 - - @staticmethod - def allowedKeywordSuffixes(value): - return { ParserKind.TAG: ['.'], - ParserKind.COMMAND: [':'], - ParserKind.LIST: [':'], - ParserKind.BOOLEAN_EXPR: [':'], - ParserKind.CUSTOM: [':', '.'] - } [value] - - @staticmethod - def str(value): - return { ParserKind.TAG: 'TAG', - ParserKind.COMMAND: 'COMMAND', - ParserKind.LIST: 'LIST', - ParserKind.BOOLEAN_EXPR: 'BOOLEAN_EXPR', - ParserKind.CUSTOM: 'CUSTOM' - } [value] + CUSTOM = 3 class IntegratedTestKeywordParser(object): @@ -787,18 +765,15 @@ class IntegratedTestKeywordParser(object): ParserKind.CUSTOM. """ def __init__(self, keyword, kind, parser=None, initial_value=None): - allowedSuffixes = ParserKind.allowedKeywordSuffixes(kind) - if len(keyword) == 0 or keyword[-1] not in allowedSuffixes: - if len(allowedSuffixes) == 1: - raise ValueError("Keyword '%s' of kind '%s' must end in '%s'" - % (keyword, ParserKind.str(kind), - allowedSuffixes[0])) - else: - raise ValueError("Keyword '%s' of kind '%s' must end in " - " one of '%s'" - % (keyword, ParserKind.str(kind), - ' '.join(allowedSuffixes))) - + if not keyword.endswith('.') and not keyword.endswith(':'): + raise ValueError("keyword '%s' must end with either '.' or ':' " + % keyword) + if keyword.endswith('.') and kind in \ + [ParserKind.LIST, ParserKind.COMMAND]: + raise ValueError("Keyword '%s' should end in ':'" % keyword) + + elif keyword.endswith(':') and kind in [ParserKind.TAG]: + raise ValueError("Keyword '%s' should end in '.'" % keyword) if parser is not None and kind != ParserKind.CUSTOM: raise ValueError("custom parsers can only be specified with " "ParserKind.CUSTOM") @@ -812,9 +787,9 @@ class IntegratedTestKeywordParser(object): self.parser = self._handleCommand elif kind == ParserKind.LIST: self.parser = self._handleList - elif kind == ParserKind.BOOLEAN_EXPR: - self.parser = self._handleBooleanExpr elif kind == ParserKind.TAG: + if not keyword.endswith('.'): + raise ValueError("keyword '%s' should end with '.'" % keyword) self.parser = self._handleTag elif kind == ParserKind.CUSTOM: if parser is None: @@ -824,12 +799,8 @@ class IntegratedTestKeywordParser(object): raise ValueError("Unknown kind '%s'" % kind) def parseLine(self, line_number, line): - try: - self.parsed_lines += [(line_number, line)] - self.value = self.parser(line_number, line, self.value) - except ValueError as e: - raise ValueError(str(e) + ("\nin %s directive on test line %d" % - (self.keyword, line_number))) + self.parsed_lines += [(line_number, line)] + self.value = self.parser(line_number, line, self.value) def getValue(self): return self.value @@ -870,38 +841,12 @@ class IntegratedTestKeywordParser(object): output.extend([s.strip() for s in line.split(',')]) return output - @staticmethod - def _handleBooleanExpr(line_number, line, output): - """A parser for BOOLEAN_EXPR type keywords""" - if output is None: - output = [] - output.extend([s.strip() for s in line.split(',')]) - # Evaluate each expression to verify syntax. - # We don't want any results, just the raised ValueError. - for s in output: - if s != '*': - BooleanExpression.evaluate(s, []) - return output - - @staticmethod - def _handleRequiresAny(line_number, line, output): - """A custom parser to transform REQUIRES-ANY: into REQUIRES:""" - - # Extract the conditions specified in REQUIRES-ANY: as written. - conditions = [] - IntegratedTestKeywordParser._handleList(line_number, line, conditions) - - # Output a `REQUIRES: a || b || c` expression in its place. - expression = ' || '.join(conditions) - IntegratedTestKeywordParser._handleBooleanExpr(line_number, - expression, output) - return output def parseIntegratedTestScript(test, additional_parsers=[], require_script=True): """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES' - and 'UNSUPPORTED' information. + 'REQUIRES-ANY' and 'UNSUPPORTED' information. If additional parsers are specified then the test is also scanned for the keywords they specify and all matches are passed to the custom parser. @@ -910,26 +855,26 @@ def parseIntegratedTestScript(test, additional_parsers=[], may be returned. This can be used for test formats where the actual script is optional or ignored. """ - - # Install the built-in keyword parsers. + # Collect the test lines from the script. + sourcepath = test.getSourcePath() script = [] + requires = [] + requires_any = [] + unsupported = [] builtin_parsers = [ IntegratedTestKeywordParser('RUN:', ParserKind.COMMAND, initial_value=script), - IntegratedTestKeywordParser('XFAIL:', ParserKind.BOOLEAN_EXPR, + IntegratedTestKeywordParser('XFAIL:', ParserKind.LIST, initial_value=test.xfails), - IntegratedTestKeywordParser('REQUIRES:', ParserKind.BOOLEAN_EXPR, - initial_value=test.requires), - IntegratedTestKeywordParser('REQUIRES-ANY:', ParserKind.CUSTOM, - IntegratedTestKeywordParser._handleRequiresAny, - initial_value=test.requires), - IntegratedTestKeywordParser('UNSUPPORTED:', ParserKind.BOOLEAN_EXPR, - initial_value=test.unsupported), + IntegratedTestKeywordParser('REQUIRES:', ParserKind.LIST, + initial_value=requires), + IntegratedTestKeywordParser('REQUIRES-ANY:', ParserKind.LIST, + initial_value=requires_any), + IntegratedTestKeywordParser('UNSUPPORTED:', ParserKind.LIST, + initial_value=unsupported), IntegratedTestKeywordParser('END.', ParserKind.TAG) ] keyword_parsers = {p.keyword: p for p in builtin_parsers} - - # Install user-defined additional parsers. for parser in additional_parsers: if not isinstance(parser, IntegratedTestKeywordParser): raise ValueError('additional parser must be an instance of ' @@ -938,9 +883,7 @@ def parseIntegratedTestScript(test, additional_parsers=[], raise ValueError("Parser for keyword '%s' already exists" % parser.keyword) keyword_parsers[parser.keyword] = parser - - # Collect the test lines from the script. - sourcepath = test.getSourcePath() + for line_number, command_type, ln in \ parseIntegratedTestScriptCommands(sourcepath, keyword_parsers.keys()): @@ -958,30 +901,46 @@ def parseIntegratedTestScript(test, additional_parsers=[], return lit.Test.Result(Test.UNRESOLVED, "Test has unterminated run lines (with '\\')") - # Enforce REQUIRES: - missing_required_features = test.getMissingRequiredFeatures() + # Check that we have the required features: + missing_required_features = [f for f in requires + if f not in test.config.available_features] if missing_required_features: msg = ', '.join(missing_required_features) return lit.Test.Result(Test.UNSUPPORTED, - "Test requires the following unavailable " - "features: %s" % msg) - - # Enforce UNSUPPORTED: - unsupported_features = test.getUnsupportedFeatures() + "Test requires the following features: %s" + % msg) + requires_any_features = [f for f in requires_any + if f in test.config.available_features] + if requires_any and not requires_any_features: + msg = ' ,'.join(requires_any) + return lit.Test.Result(Test.UNSUPPORTED, + "Test requires any of the following features: " + "%s" % msg) + unsupported_features = [f for f in unsupported + if f in test.config.available_features] if unsupported_features: msg = ', '.join(unsupported_features) return lit.Test.Result( Test.UNSUPPORTED, - "Test does not support the following features " - "and/or targets: %s" % msg) - - # Enforce limit_to_features. - if not test.isWithinFeatureLimits(): - msg = ', '.join(test.config.limit_to_features) - return lit.Test.Result(Test.UNSUPPORTED, - "Test does not require any of the features " - "specified in limit_to_features: %s" % msg) + "Test is unsupported with the following features: %s" % msg) + unsupported_targets = [f for f in unsupported + if f in test.suite.config.target_triple] + if unsupported_targets: + return lit.Test.Result( + Test.UNSUPPORTED, + "Test is unsupported with the following triple: %s" % ( + test.suite.config.target_triple,)) + + if test.config.limit_to_features: + # Check that we have one of the limit_to_features features in requires. + limit_to_features_tests = [f for f in test.config.limit_to_features + if f in requires] + if not limit_to_features_tests: + msg = ', '.join(test.config.limit_to_features) + return lit.Test.Result( + Test.UNSUPPORTED, + "Test requires one of the limit_to_features features %s" % msg) return script diff --git a/utils/lit/tests/Inputs/shtest-format/requires-missing.txt b/utils/lit/tests/Inputs/shtest-format/requires-missing.txt index d643e57edca..9e6648d8b8f 100644 --- a/utils/lit/tests/Inputs/shtest-format/requires-missing.txt +++ b/utils/lit/tests/Inputs/shtest-format/requires-missing.txt @@ -1,5 +1,2 @@ -# REQUIRES with a false clause. Test should not run. -REQUIRES: true -REQUIRES: a-missing-feature, true -REQUIRES: true -RUN: false +RUN: true +REQUIRES: a-missing-feature diff --git a/utils/lit/tests/Inputs/shtest-format/requires-present.txt b/utils/lit/tests/Inputs/shtest-format/requires-present.txt index 9fcbdca69be..064f7074a76 100644 --- a/utils/lit/tests/Inputs/shtest-format/requires-present.txt +++ b/utils/lit/tests/Inputs/shtest-format/requires-present.txt @@ -1,4 +1,2 @@ -# REQUIRES with only true clauses. Test should run. -REQUIRES: a-present-feature, true, !not-true -REQUIRES: true RUN: true +REQUIRES: a-present-feature diff --git a/utils/lit/tests/Inputs/shtest-format/requires-star.txt b/utils/lit/tests/Inputs/shtest-format/requires-star.txt deleted file mode 100644 index 5566d8b15b0..00000000000 --- a/utils/lit/tests/Inputs/shtest-format/requires-star.txt +++ /dev/null @@ -1,3 +0,0 @@ -# '*' only works in XFAIL -REQUIRES: * -RUN: false diff --git a/utils/lit/tests/Inputs/shtest-format/requires-triple.txt b/utils/lit/tests/Inputs/shtest-format/requires-triple.txt deleted file mode 100644 index 6470bf40414..00000000000 --- a/utils/lit/tests/Inputs/shtest-format/requires-triple.txt +++ /dev/null @@ -1,3 +0,0 @@ -# REQUIRES line that uses target triple, which doesn't work. Test should not run -REQUIRES: x86_64 -RUN: false diff --git a/utils/lit/tests/Inputs/shtest-format/unsupported-expr-false.txt b/utils/lit/tests/Inputs/shtest-format/unsupported-expr-false.txt deleted file mode 100644 index 00c6160a367..00000000000 --- a/utils/lit/tests/Inputs/shtest-format/unsupported-expr-false.txt +++ /dev/null @@ -1,9 +0,0 @@ -# UNSUPPORTED with only false clauses. Test should run. -UNSUPPORTED: false -UNSUPPORTED: false, not-true -UNSUPPORTED: false -UNSUPPORTED: still-not-true -UNSUPPORTED: false -UNSUPPORTED: false -UNSUPPORTED: false -RUN: true diff --git a/utils/lit/tests/Inputs/shtest-format/unsupported-expr-true.txt b/utils/lit/tests/Inputs/shtest-format/unsupported-expr-true.txt deleted file mode 100644 index f48ba7b2c2d..00000000000 --- a/utils/lit/tests/Inputs/shtest-format/unsupported-expr-true.txt +++ /dev/null @@ -1,4 +0,0 @@ -# UNSUPPORTED with a true clause. Test should not run. -UNSUPPORTED: false -UNSUPPORTED: false, false, false, _64-unk && a-present-feature, false -RUN: false diff --git a/utils/lit/tests/Inputs/shtest-format/unsupported-star.txt b/utils/lit/tests/Inputs/shtest-format/unsupported-star.txt deleted file mode 100644 index 16630207dac..00000000000 --- a/utils/lit/tests/Inputs/shtest-format/unsupported-star.txt +++ /dev/null @@ -1,3 +0,0 @@ -# '*' only works in XFAIL -UNSUPPORTED: * -RUN: false diff --git a/utils/lit/tests/Inputs/shtest-format/xfail-expr-false.txt b/utils/lit/tests/Inputs/shtest-format/xfail-expr-false.txt deleted file mode 100644 index 83b0de1621d..00000000000 --- a/utils/lit/tests/Inputs/shtest-format/xfail-expr-false.txt +++ /dev/null @@ -1,3 +0,0 @@ -# XFAIL with only false clauses. Test should run. -XFAIL: false, a-missing-feature || ! a-present-feature || ! x86_64, false -RUN: true diff --git a/utils/lit/tests/Inputs/shtest-format/xfail-expr-true.txt b/utils/lit/tests/Inputs/shtest-format/xfail-expr-true.txt deleted file mode 100644 index 3c197484897..00000000000 --- a/utils/lit/tests/Inputs/shtest-format/xfail-expr-true.txt +++ /dev/null @@ -1,4 +0,0 @@ -# XFAIL with a true clause. Test should not run. -XFAIL: false -XFAIL: false, a-present-feature && ! a-missing-feature && x86_64 -RUN: false diff --git a/utils/lit/tests/boolean-parsing.py b/utils/lit/tests/boolean-parsing.py deleted file mode 100644 index 372a94d2332..00000000000 --- a/utils/lit/tests/boolean-parsing.py +++ /dev/null @@ -1,4 +0,0 @@ -# Test the boolean expression parser -# used for REQUIRES and UNSUPPORTED and XFAIL - -# RUN: %{python} -m lit.BooleanExpression diff --git a/utils/lit/tests/shtest-format.py b/utils/lit/tests/shtest-format.py index 37e3e1c1262..20884f8c485 100644 --- a/utils/lit/tests/shtest-format.py +++ b/utils/lit/tests/shtest-format.py @@ -50,14 +50,7 @@ # CHECK: PASS: shtest-format :: requires-any-present.txt # CHECK: UNSUPPORTED: shtest-format :: requires-missing.txt # CHECK: PASS: shtest-format :: requires-present.txt -# CHECK: UNRESOLVED: shtest-format :: requires-star.txt -# CHECK: UNSUPPORTED: shtest-format :: requires-triple.txt -# CHECK: PASS: shtest-format :: unsupported-expr-false.txt -# CHECK: UNSUPPORTED: shtest-format :: unsupported-expr-true.txt -# CHECK: UNRESOLVED: shtest-format :: unsupported-star.txt # CHECK: UNSUPPORTED: shtest-format :: unsupported_dir/some-test.txt -# CHECK: PASS: shtest-format :: xfail-expr-false.txt -# CHECK: XFAIL: shtest-format :: xfail-expr-true.txt # CHECK: XFAIL: shtest-format :: xfail-feature.txt # CHECK: XFAIL: shtest-format :: xfail-target.txt # CHECK: XFAIL: shtest-format :: xfail.txt @@ -77,9 +70,9 @@ # CHECK: shtest-format :: external_shell/fail_with_bad_encoding.txt # CHECK: shtest-format :: fail.txt -# CHECK: Expected Passes : 7 -# CHECK: Expected Failures : 4 -# CHECK: Unsupported Tests : 5 -# CHECK: Unresolved Tests : 3 +# CHECK: Expected Passes : 5 +# CHECK: Expected Failures : 3 +# CHECK: Unsupported Tests : 3 +# CHECK: Unresolved Tests : 1 # CHECK: Unexpected Passes : 1 # CHECK: Unexpected Failures: 3 diff --git a/utils/lit/tests/unit/TestRunner.py b/utils/lit/tests/unit/TestRunner.py index ed0affa2832..ff11834fed7 100644 --- a/utils/lit/tests/unit/TestRunner.py +++ b/utils/lit/tests/unit/TestRunner.py @@ -108,63 +108,6 @@ class TestIntegratedTestKeywordParser(unittest.TestCase): value = custom_parser.getValue() self.assertItemsEqual(value, ['a', 'b', 'c']) - def test_bad_keywords(self): - def custom_parse(line_number, line, output): - return output - - try: - IntegratedTestKeywordParser("TAG_NO_SUFFIX", ParserKind.TAG), - self.fail("TAG_NO_SUFFIX failed to raise an exception") - except ValueError as e: - pass - except BaseException as e: - self.fail("TAG_NO_SUFFIX raised the wrong exception: %r" % e) - - try: - IntegratedTestKeywordParser("TAG_WITH_COLON:", ParserKind.TAG), - self.fail("TAG_WITH_COLON: failed to raise an exception") - except ValueError as e: - pass - except BaseException as e: - self.fail("TAG_WITH_COLON: raised the wrong exception: %r" % e) - - try: - IntegratedTestKeywordParser("LIST_WITH_DOT.", ParserKind.LIST), - self.fail("LIST_WITH_DOT. failed to raise an exception") - except ValueError as e: - pass - except BaseException as e: - self.fail("LIST_WITH_DOT. raised the wrong exception: %r" % e) - - try: - IntegratedTestKeywordParser("CUSTOM_NO_SUFFIX", - ParserKind.CUSTOM, custom_parse), - self.fail("CUSTOM_NO_SUFFIX failed to raise an exception") - except ValueError as e: - pass - except BaseException as e: - self.fail("CUSTOM_NO_SUFFIX raised the wrong exception: %r" % e) - - # Both '.' and ':' are allowed for CUSTOM keywords. - try: - IntegratedTestKeywordParser("CUSTOM_WITH_DOT.", - ParserKind.CUSTOM, custom_parse), - except BaseException as e: - self.fail("CUSTOM_WITH_DOT. raised an exception: %r" % e) - try: - IntegratedTestKeywordParser("CUSTOM_WITH_COLON:", - ParserKind.CUSTOM, custom_parse), - except BaseException as e: - self.fail("CUSTOM_WITH_COLON: raised an exception: %r" % e) - - try: - IntegratedTestKeywordParser("CUSTOM_NO_PARSER:", - ParserKind.CUSTOM), - self.fail("CUSTOM_NO_PARSER: failed to raise an exception") - except ValueError as e: - pass - except BaseException as e: - self.fail("CUSTOM_NO_PARSER: raised the wrong exception: %r" % e) if __name__ == '__main__': TestIntegratedTestKeywordParser.load_keyword_parser_lit_tests() |