[llvm] fbdcfcd - [lit] Provide extension API for custom result categories
Julian Lettner via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 29 19:48:18 PDT 2020
Author: Julian Lettner
Date: 2020-04-29T19:45:55-07:00
New Revision: fbdcfcd4c392c487c411632b951f4aada351154e
URL: https://github.com/llvm/llvm-project/commit/fbdcfcd4c392c487c411632b951f4aada351154e
DIFF: https://github.com/llvm/llvm-project/commit/fbdcfcd4c392c487c411632b951f4aada351154e.diff
LOG: [lit] Provide extension API for custom result categories
The lnt test suite defines custom result codes [1]. Support those via
an extension API instead of "by accident", which should offer the
advantage of properly handling them when we print test results.
[1] https://reviews.llvm.org/D77986
Reviewed By: Meinersbur
Differential Revision: https://reviews.llvm.org/D78164
Added:
llvm/utils/lit/tests/Inputs/custom-result-category/format.py
llvm/utils/lit/tests/Inputs/custom-result-category/lit.cfg
llvm/utils/lit/tests/Inputs/custom-result-category/test1.txt
llvm/utils/lit/tests/Inputs/custom-result-category/test2.txt
llvm/utils/lit/tests/custom-result-category.py
Modified:
llvm/utils/lit/lit/main.py
llvm/utils/lit/tests/Inputs/shtest-inject/lit.cfg
Removed:
################################################################################
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 25f15107a211..17382da52bfb 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -252,29 +252,34 @@ def print_histogram(tests):
lit.util.printHistogram(test_times, title='Tests')
+def add_result_category(result_code, label):
+ assert isinstance(result_code, lit.Test.ResultCode)
+ category = (result_code, "%s Tests" % label, label)
+ result_codes.append(category)
+
+
# Status code, summary label, group label
-failure_codes = [
+result_codes = [
+ # Passes
+ (lit.Test.SKIPPED, 'Skipped Tests', 'Skipped'),
+ (lit.Test.UNSUPPORTED, 'Unsupported Tests', 'Unsupported'),
+ (lit.Test.PASS, 'Expected Passes', ''),
+ (lit.Test.FLAKYPASS, 'Passes With Retry', ''),
+ (lit.Test.XFAIL, 'Expected Failures', 'Expected Failing'),
+ # Failures
(lit.Test.UNRESOLVED, 'Unresolved Tests', 'Unresolved'),
(lit.Test.TIMEOUT, 'Individual Timeouts', 'Timed Out'),
(lit.Test.FAIL, 'Unexpected Failures', 'Failing'),
(lit.Test.XPASS, 'Unexpected Passes', 'Unexpected Passing')
]
-all_codes = [
- (lit.Test.SKIPPED, 'Skipped Tests', 'Skipped'),
- (lit.Test.UNSUPPORTED, 'Unsupported Tests', 'Unsupported'),
- (lit.Test.PASS, 'Expected Passes', ''),
- (lit.Test.FLAKYPASS, 'Passes With Retry', ''),
- (lit.Test.XFAIL, 'Expected Failures', 'Expected Failing'),
-] + failure_codes
-
def print_results(tests, elapsed, opts):
- tests_by_code = {code: [] for (code, _, _) in all_codes}
+ tests_by_code = {code: [] for (code, _, _) in result_codes}
for test in tests:
tests_by_code[test.result.code].append(test)
- for (code, _, group_label) in all_codes:
+ for (code, _, group_label) in result_codes:
print_group(code, group_label, tests_by_code[code], opts)
print_summary(tests_by_code, opts.quiet, elapsed)
@@ -300,7 +305,7 @@ def print_summary(tests_by_code, quiet, elapsed):
if not quiet:
print('\nTesting Time: %.2fs' % elapsed)
- codes = failure_codes if quiet else all_codes
+ codes = [c for c in result_codes if not quiet or c.isFailure]
groups = [(label, len(tests_by_code[code])) for code, label, _ in codes]
groups = [(label, count) for label, count in groups if count]
if not groups:
diff --git a/llvm/utils/lit/tests/Inputs/custom-result-category/format.py b/llvm/utils/lit/tests/Inputs/custom-result-category/format.py
new file mode 100644
index 000000000000..b0c97ec71bb9
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/custom-result-category/format.py
@@ -0,0 +1,18 @@
+import lit
+import lit.formats
+
+CUSTOM_PASS = lit.Test.ResultCode('CUSTOM_PASS', False)
+CUSTOM_FAILURE = lit.Test.ResultCode('CUSTOM_FAILURE', True)
+
+lit.main.add_result_category(CUSTOM_PASS, "My Passed")
+lit.main.add_result_category(CUSTOM_FAILURE, "My Failed")
+
+
+class MyFormat(lit.formats.ShTest):
+ def execute(self, test, lit_config):
+ result = super(MyFormat, self).execute(test, lit_config)
+ if result.code.isFailure:
+ result.code = CUSTOM_FAILURE
+ else:
+ result.code = CUSTOM_PASS
+ return result
diff --git a/llvm/utils/lit/tests/Inputs/custom-result-category/lit.cfg b/llvm/utils/lit/tests/Inputs/custom-result-category/lit.cfg
new file mode 100644
index 000000000000..842ea612ebf0
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/custom-result-category/lit.cfg
@@ -0,0 +1,10 @@
+import lit
+import site
+site.addsitedir(os.path.dirname(__file__))
+import format
+
+config.name = 'custom-result-category'
+config.suffixes = ['.txt']
+config.test_format = format.MyFormat()
+config.test_source_root = None
+config.test_exec_root = None
diff --git a/llvm/utils/lit/tests/Inputs/custom-result-category/test1.txt b/llvm/utils/lit/tests/Inputs/custom-result-category/test1.txt
new file mode 100644
index 000000000000..b80b60b7a279
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/custom-result-category/test1.txt
@@ -0,0 +1 @@
+# RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/custom-result-category/test2.txt b/llvm/utils/lit/tests/Inputs/custom-result-category/test2.txt
new file mode 100644
index 000000000000..49932c3006e1
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/custom-result-category/test2.txt
@@ -0,0 +1 @@
+# RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/shtest-inject/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-inject/lit.cfg
index dfec55a3d427..755edf8b5bfc 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-inject/lit.cfg
+++ b/llvm/utils/lit/tests/Inputs/shtest-inject/lit.cfg
@@ -3,7 +3,7 @@ import lit
preamble_commands = [
'echo "THIS WAS"',
'echo "INJECTED"'
-];
+]
config.name = 'shtest-inject'
config.suffixes = ['.txt']
diff --git a/llvm/utils/lit/tests/custom-result-category.py b/llvm/utils/lit/tests/custom-result-category.py
new file mode 100644
index 000000000000..49d0bbee4f0d
--- /dev/null
+++ b/llvm/utils/lit/tests/custom-result-category.py
@@ -0,0 +1,14 @@
+# Test lit.main.add_result_category() extension API.
+
+# RUN: not %{lit} -j 1 %{inputs}/custom-result-category | FileCheck %s
+
+# CHECK: CUSTOM_PASS: custom-result-category :: test1.txt
+# CHECK: CUSTOM_FAILURE: custom-result-category :: test2.txt
+
+# TODO(yln): Passing tests shouldn't be printed by default.
+# CHECK: My Passed Tests (1)
+# CHECK: My Failed Tests (1)
+# CHECK: custom-result-category :: test2.txt
+
+# CHECK: My Passed Tests: 1
+# CHECK: My Failed Tests: 1
More information about the llvm-commits
mailing list