[llvm] dac21fd - [lit] Add an option to print all features used in tests
Louis Dionne via llvm-commits
llvm-commits at lists.llvm.org
Fri May 29 04:00:17 PDT 2020
Author: Louis Dionne
Date: 2020-05-29T07:00:05-04:00
New Revision: dac21fd29cd2ae2b979a276747ad5ad82fca09bf
URL: https://github.com/llvm/llvm-project/commit/dac21fd29cd2ae2b979a276747ad5ad82fca09bf
DIFF: https://github.com/llvm/llvm-project/commit/dac21fd29cd2ae2b979a276747ad5ad82fca09bf.diff
LOG: [lit] Add an option to print all features used in tests
Lit test suites can tend to accumulate annotations that are not necessarily
relevant as time goes by, for example XFAILS on old compilers or platforms.
To help spot old annotations that can be cleaned up, it can be useful to
look at all features used inside a test suite.
This commit adds a new Lit option '--show-used-features' that prints all
the features used in XFAIL, REQUIRES and UNSUPPORTED of all tests that
are discovered.
Differential Revision: https://reviews.llvm.org/D78589
Added:
llvm/utils/lit/tests/Inputs/show-used-features/lit.cfg
llvm/utils/lit/tests/Inputs/show-used-features/mixed.txt
llvm/utils/lit/tests/Inputs/show-used-features/requires.txt
llvm/utils/lit/tests/Inputs/show-used-features/unsupported.txt
llvm/utils/lit/tests/Inputs/show-used-features/xfail.txt
llvm/utils/lit/tests/show-used-features.py
Modified:
llvm/utils/lit/lit/BooleanExpression.py
llvm/utils/lit/lit/Test.py
llvm/utils/lit/lit/TestRunner.py
llvm/utils/lit/lit/cl_arguments.py
llvm/utils/lit/lit/main.py
Removed:
################################################################################
diff --git a/llvm/utils/lit/lit/BooleanExpression.py b/llvm/utils/lit/lit/BooleanExpression.py
index 3eb5060de3e3..34e07fc1b8e5 100644
--- a/llvm/utils/lit/lit/BooleanExpression.py
+++ b/llvm/utils/lit/lit/BooleanExpression.py
@@ -79,9 +79,10 @@ def expect(self, t):
raise ValueError("expected: %s\nhave: %s" %
(self.quote(t), self.quote(self.token)))
- def isIdentifier(self, t):
- if (t is BooleanExpression.END or t == '&&' or t == '||' or
- t == '!' or t == '(' or t == ')'):
+ @staticmethod
+ def isIdentifier(token):
+ if (token is BooleanExpression.END or token == '&&' or token == '||' or
+ token == '!' or token == '(' or token == ')'):
return False
return True
@@ -92,7 +93,7 @@ def parseNOT(self):
elif self.accept('('):
self.parseOR()
self.expect(')')
- elif not self.isIdentifier(self.token):
+ elif not BooleanExpression.isIdentifier(self.token):
raise ValueError("expected: '!' or '(' or identifier\nhave: %s" %
self.quote(self.token))
else:
@@ -191,7 +192,7 @@ def checkException(self, expr, error):
"actual error was:\n%s\n" +
"expected error was:\n%s\n") % (expr, e, error))
except BaseException as e:
- self.fail(("expression %r caused the wrong exception; actual " +
+ self.fail(("expression %r caused the wrong exception; actual " +
"exception was: \n%r") % (expr, e))
def test_errors(self):
diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py
index 627785829efc..4c94e9806a16 100644
--- a/llvm/utils/lit/lit/Test.py
+++ b/llvm/utils/lit/lit/Test.py
@@ -1,3 +1,4 @@
+import itertools
import os
from xml.sax.saxutils import quoteattr
from json import JSONEncoder
@@ -162,7 +163,7 @@ def addMicroResult(self, name, microResult):
addMicroResult(microResult)
Attach a micro-test result to the test result, with the given name and
- result. It is an error to attempt to attach a micro-test with the
+ result. It is an error to attempt to attach a micro-test with the
same name multiple times.
Each micro-test result must be an instance of the Result class.
@@ -359,6 +360,26 @@ def getUnsupportedFeatures(self):
except ValueError as e:
raise ValueError('Error in UNSUPPORTED list:\n%s' % str(e))
+ def getUsedFeatures(self):
+ """
+ getUsedFeatures() -> list of strings
+
+ Returns a list of all features appearing in XFAIL, UNSUPPORTED and
+ REQUIRES annotations for this test.
+ """
+ import lit.TestRunner
+ parsed = lit.TestRunner._parseKeywords(self.getSourcePath(), require_script=False)
+ feature_keywords = ('UNSUPPORTED:', 'REQUIRES:', 'XFAIL:')
+ boolean_expressions = itertools.chain.from_iterable(
+ parsed[k] or [] for k in feature_keywords
+ )
+ tokens = itertools.chain.from_iterable(
+ BooleanExpression.tokenize(expr) for expr in
+ boolean_expressions if expr != '*'
+ )
+ identifiers = set(filter(BooleanExpression.isIdentifier, tokens))
+ return identifiers
+
def isEarlyTest(self):
"""
isEarlyTest() -> bool
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index 6c92a2bc26c4..edcb696a8b16 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -1373,31 +1373,26 @@ def _handleBooleanExpr(line_number, line, output):
BooleanExpression.evaluate(s, [])
return output
-def parseIntegratedTestScript(test, additional_parsers=[],
- require_script=True):
- """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
- script and extract the lines to 'RUN' as well as 'XFAIL', 'REQUIRES',
- 'UNSUPPORTED' and 'ALLOW_RETRIES' information.
- If additional parsers are specified then the test is also scanned for the
- keywords they specify and all matches are passed to the custom parser.
+def _parseKeywords(sourcepath, additional_parsers=[],
+ require_script=True):
+ """_parseKeywords
- If 'require_script' is False an empty script
- may be returned. This can be used for test formats where the actual script
- is optional or ignored.
- """
+ Scan an LLVM/Clang style integrated test script and extract all the lines
+ pertaining to a special parser. This includes 'RUN', 'XFAIL', 'REQUIRES',
+ 'UNSUPPORTED' and 'ALLOW_RETRIES', as well as other specified custom
+ parsers.
+ Returns a dictionary mapping each custom parser to its value after
+ parsing the test.
+ """
# Install the built-in keyword parsers.
script = []
builtin_parsers = [
- IntegratedTestKeywordParser('RUN:', ParserKind.COMMAND,
- initial_value=script),
- IntegratedTestKeywordParser('XFAIL:', ParserKind.BOOLEAN_EXPR,
- initial_value=test.xfails),
- IntegratedTestKeywordParser('REQUIRES:', ParserKind.BOOLEAN_EXPR,
- initial_value=test.requires),
- IntegratedTestKeywordParser('UNSUPPORTED:', ParserKind.BOOLEAN_EXPR,
- initial_value=test.unsupported),
+ IntegratedTestKeywordParser('RUN:', ParserKind.COMMAND, initial_value=script),
+ IntegratedTestKeywordParser('XFAIL:', ParserKind.BOOLEAN_EXPR),
+ IntegratedTestKeywordParser('REQUIRES:', ParserKind.BOOLEAN_EXPR),
+ IntegratedTestKeywordParser('UNSUPPORTED:', ParserKind.BOOLEAN_EXPR),
IntegratedTestKeywordParser('ALLOW_RETRIES:', ParserKind.INTEGER),
IntegratedTestKeywordParser('END.', ParserKind.TAG)
]
@@ -1414,7 +1409,6 @@ def parseIntegratedTestScript(test, additional_parsers=[],
keyword_parsers[parser.keyword] = parser
# Collect the test lines from the script.
- sourcepath = test.getSourcePath()
for line_number, command_type, ln in \
parseIntegratedTestScriptCommands(sourcepath,
keyword_parsers.keys()):
@@ -1441,6 +1435,37 @@ def parseIntegratedTestScript(test, additional_parsers=[],
if value and value[-1][-1] == '\\':
raise ValueError("Test has unterminated %s lines (with '\\')" % key)
+ # Make sure there's at most one ALLOW_RETRIES: line
+ allowed_retries = keyword_parsers['ALLOW_RETRIES:'].getValue()
+ if allowed_retries and len(allowed_retries) > 1:
+ return lit.Test.Result(Test.UNRESOLVED,
+ "Test has more than one ALLOW_RETRIES lines")
+
+ return {p.keyword: p.getValue() for p in keyword_parsers.values()}
+
+
+def parseIntegratedTestScript(test, additional_parsers=[],
+ require_script=True):
+ """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
+ script and extract the lines to 'RUN' as well as 'XFAIL', 'REQUIRES',
+ 'UNSUPPORTED' and 'ALLOW_RETRIES' information into the given test.
+
+ If additional parsers are specified then the test is also scanned for the
+ keywords they specify and all matches are passed to the custom parser.
+
+ If 'require_script' is False an empty script
+ may be returned. This can be used for test formats where the actual script
+ is optional or ignored.
+ """
+ # Parse the test sources and extract test properties
+ parsed = _parseKeywords(test.getSourcePath(), additional_parsers, require_script)
+ script = parsed['RUN:'] or []
+ test.xfails = parsed['XFAIL:'] or []
+ test.requires = parsed['REQUIRES:'] or []
+ test.unsupported = parsed['UNSUPPORTED:'] or []
+ if parsed['ALLOW_RETRIES:']:
+ test.allowed_retries = parsed['ALLOW_RETRIES:'][0]
+
# Enforce REQUIRES:
missing_required_features = test.getMissingRequiredFeatures()
if missing_required_features:
@@ -1458,14 +1483,6 @@ def parseIntegratedTestScript(test, additional_parsers=[],
"Test does not support the following features "
"and/or targets: %s" % msg)
- # Handle ALLOW_RETRIES:
- allowed_retries = keyword_parsers['ALLOW_RETRIES:'].getValue()
- if allowed_retries:
- if len(allowed_retries) > 1:
- return lit.Test.Result(Test.UNRESOLVED,
- "Test has more than one ALLOW_RETRIES lines")
- test.allowed_retries = allowed_retries[0]
-
# Enforce limit_to_features.
if not test.isWithinFeatureLimits():
msg = ', '.join(test.config.limit_to_features)
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 06b1313cb8bb..6bbaa4203b47 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -157,6 +157,9 @@ def parse_args():
debug_group.add_argument("--show-tests",
help="Show all discovered tests and exit",
action="store_true")
+ debug_group.add_argument("--show-used-features",
+ help="Show all features used in the test suite (in XFAIL, UNSUPPORTED and REQUIRES) and exit",
+ action="store_true")
# LIT is special: environment variables override command line arguments.
env_args = shlex.split(os.environ.get("LIT_OPTS", ""))
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index d2958590aa8a..6c423167ff4c 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -4,6 +4,7 @@
See lit.pod for more information.
"""
+import itertools
import os
import platform
import sys
@@ -47,6 +48,11 @@ def main(builtin_params={}):
print_discovered(discovered_tests, opts.show_suites, opts.show_tests)
sys.exit(0)
+ if opts.show_used_features:
+ features = set(itertools.chain.from_iterable(t.getUsedFeatures() for t in discovered_tests))
+ print(' '.join(sorted(features)))
+ sys.exit(0)
+
# Command line overrides configuration for maxIndividualTestTime.
if opts.maxIndividualTestTime is not None: # `not None` is important (default: 0)
if opts.maxIndividualTestTime != lit_config.maxIndividualTestTime:
@@ -127,7 +133,6 @@ def print_discovered(tests, show_suites, show_tests):
tests.sort(key=lit.reports.by_suite_and_test_path)
if show_suites:
- import itertools
tests_by_suite = itertools.groupby(tests, lambda t: t.suite)
print('-- Test Suites --')
for suite, test_iter in tests_by_suite:
diff --git a/llvm/utils/lit/tests/Inputs/show-used-features/lit.cfg b/llvm/utils/lit/tests/Inputs/show-used-features/lit.cfg
new file mode 100644
index 000000000000..7ee2154d2e19
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-used-features/lit.cfg
@@ -0,0 +1,6 @@
+import lit.formats
+config.name = 'show-used-features'
+config.suffixes = ['.txt']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
diff --git a/llvm/utils/lit/tests/Inputs/show-used-features/mixed.txt b/llvm/utils/lit/tests/Inputs/show-used-features/mixed.txt
new file mode 100644
index 000000000000..1de0f7442a08
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-used-features/mixed.txt
@@ -0,0 +1,4 @@
+
+// REQUIRES: my-require-feature-2 || my-require-feature-3
+// UNSUPPORTED: my-unsupported-feature-2, my-unsupported-feature-3
+// XFAIL: my-xfail-feature-2, my-xfail-feature-3
diff --git a/llvm/utils/lit/tests/Inputs/show-used-features/requires.txt b/llvm/utils/lit/tests/Inputs/show-used-features/requires.txt
new file mode 100644
index 000000000000..3e550f4495fb
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-used-features/requires.txt
@@ -0,0 +1,2 @@
+
+// REQUIRES: my-require-feature-1
diff --git a/llvm/utils/lit/tests/Inputs/show-used-features/unsupported.txt b/llvm/utils/lit/tests/Inputs/show-used-features/unsupported.txt
new file mode 100644
index 000000000000..b19582624abc
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-used-features/unsupported.txt
@@ -0,0 +1,2 @@
+
+// UNSUPPORTED: my-unsupported-feature-1
diff --git a/llvm/utils/lit/tests/Inputs/show-used-features/xfail.txt b/llvm/utils/lit/tests/Inputs/show-used-features/xfail.txt
new file mode 100644
index 000000000000..f8cde28ab4c7
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-used-features/xfail.txt
@@ -0,0 +1,2 @@
+
+// XFAIL: my-xfail-feature-1
diff --git a/llvm/utils/lit/tests/show-used-features.py b/llvm/utils/lit/tests/show-used-features.py
new file mode 100644
index 000000000000..069ee0819670
--- /dev/null
+++ b/llvm/utils/lit/tests/show-used-features.py
@@ -0,0 +1,6 @@
+# Check that --show-used-features works correctly.
+#
+# RUN: %{lit} %{inputs}/show-used-features --show-used-features | FileCheck %s
+# CHECK: my-require-feature-1 my-require-feature-2 my-require-feature-3
+# CHECK: my-unsupported-feature-1 my-unsupported-feature-2 my-unsupported-feature-3
+# CHECK: my-xfail-feature-1 my-xfail-feature-2 my-xfail-feature-3
More information about the llvm-commits
mailing list