[Lldb-commits] [lldb] 3eea082 - [lldb][tests] Make it possible to expect failure for a whole category

[Tatyana Krasnukha via lldb-commits]

Author: Tatyana Krasnukha
Date: 2020-01-10T17:37:55+03:00
New Revision: 3eea082535e232b35e6b2dab45dd81728b2ae7f4

URL: https://github.com/llvm/llvm-project/commit/3eea082535e232b35e6b2dab45dd81728b2ae7f4
DIFF: https://github.com/llvm/llvm-project/commit/3eea082535e232b35e6b2dab45dd81728b2ae7f4.diff

LOG: [lldb][tests] Make it possible to expect failure for a whole category

There already are decorators and "--excluded" option to mark test-cases/files
as expected to fail. However, when a new test file is added and it which relates
to a feature that a target doesn't support, this requires either adding decorators
to that file or modifying the file provided as "--excluded" option value.

The purpose of this patch is to avoid any modifications in such cases.
E.g. if a target doesn't support "watchpoints" and passes "--xfail-category watchpoint"
to dotest, a testing job will not fail after a new watchpoint-related test file is added.

Differential Revision: https://reviews.llvm.org/D71906

Added: 
    

Modified: 
    lldb/packages/Python/lldbsuite/test/configuration.py
    lldb/packages/Python/lldbsuite/test/dotest.py
    lldb/packages/Python/lldbsuite/test/dotest_args.py
    lldb/packages/Python/lldbsuite/test/test_result.py

Removed: 
    


################################################################################
diff  --git a/lldb/packages/Python/lldbsuite/test/configuration.py b/lldb/packages/Python/lldbsuite/test/configuration.py
index 21d527a90478..0fc831d11730 100644
--- a/lldb/packages/Python/lldbsuite/test/configuration.py
+++ b/lldb/packages/Python/lldbsuite/test/configuration.py
@@ -30,6 +30,8 @@
 use_categories = False
 # Categories we want to skip
 skip_categories = ["darwin-log"]
+# Categories we expect to fail
+xfail_categories = []
 # use this to track per-category failures
 failures_per_category = {}
 

diff  --git a/lldb/packages/Python/lldbsuite/test/dotest.py b/lldb/packages/Python/lldbsuite/test/dotest.py
index e3f204828271..46600559286a 100644
--- a/lldb/packages/Python/lldbsuite/test/dotest.py
+++ b/lldb/packages/Python/lldbsuite/test/dotest.py
@@ -330,6 +330,10 @@ def parseOptionsAndInitTestdirs():
         configuration.skip_categories += test_categories.validate(
             args.skip_categories, False)
 
+    if args.xfail_categories:
+        configuration.xfail_categories += test_categories.validate(
+            args.xfail_categories, False)
+
     if args.E:
         os.environ['CFLAGS_EXTRAS'] = args.E
 

diff  --git a/lldb/packages/Python/lldbsuite/test/dotest_args.py b/lldb/packages/Python/lldbsuite/test/dotest_args.py
index 9bcebc58cd22..41c96e3979cd 100644
--- a/lldb/packages/Python/lldbsuite/test/dotest_args.py
+++ b/lldb/packages/Python/lldbsuite/test/dotest_args.py
@@ -83,6 +83,12 @@ def create_parser():
         action='append',
         dest='skip_categories',
         help=textwrap.dedent('''Specify categories of test cases to skip. Takes precedence over -G. Can be specified more than once.'''))
+    group.add_argument(
+        '--xfail-category',
+        metavar='category',
+        action='append',
+        dest='xfail_categories',
+        help=textwrap.dedent('''Specify categories of test cases that are expected to fail. Can be specified more than once.'''))
 
     # Configuration options
     group = parser.add_argument_group('Configuration options')

diff  --git a/lldb/packages/Python/lldbsuite/test/test_result.py b/lldb/packages/Python/lldbsuite/test/test_result.py
index 193a64c5168a..7e13e09d9bf0 100644
--- a/lldb/packages/Python/lldbsuite/test/test_result.py
+++ b/lldb/packages/Python/lldbsuite/test/test_result.py
@@ -164,6 +164,10 @@ def checkExclusion(self, exclusion_list, name):
                     return True
         return False
 
+    def checkCategoryExclusion(self, exclusion_list, test):
+        return not set(exclusion_list).isdisjoint(
+            self.getCategoriesForTest(test))
+
     def startTest(self, test):
         if configuration.shouldSkipBecauseOfCategories(
                 self.getCategoriesForTest(test)):
@@ -182,8 +186,10 @@ def startTest(self, test):
                 EventBuilder.event_for_start(test))
 
     def addSuccess(self, test):
-        if self.checkExclusion(
-                configuration.xfail_tests, test.id()):
+        if (self.checkExclusion(
+                configuration.xfail_tests, test.id()) or
+            self.checkCategoryExclusion(
+                configuration.xfail_categories, test)):
             self.addUnexpectedSuccess(test, None)
             return
 
@@ -245,8 +251,10 @@ def addCleanupError(self, test, err):
                     test, err))
 
     def addFailure(self, test, err):
-        if self.checkExclusion(
-                configuration.xfail_tests, test.id()):
+        if (self.checkExclusion(
+                configuration.xfail_tests, test.id()) or
+            self.checkCategoryExclusion(
+                configuration.xfail_categories, test)):
             self.addExpectedFailure(test, err, None)
             return