[Lldb-commits] [lldb] r282298 - Allow for tests to be disabled at runtime
Francis Ricci via lldb-commits
lldb-commits at lists.llvm.org
Fri Sep 23 14:32:47 PDT 2016
Author: fjricci
Date: Fri Sep 23 16:32:47 2016
New Revision: 282298
URL: http://llvm.org/viewvc/llvm-project?rev=282298&view=rev
Log:
Allow for tests to be disabled at runtime
Summary:
The current implementation of the test suite allows the user to run
a certain subset of tests using '-p', but does not allow the inverse,
where a user wants to run all but some number of known failing tests.
Implement this functionality.
Reviewers: labath, zturner, tfiala
Subscribers: jingham, sas, lldb-commits
Differential Revision: https://reviews.llvm.org/D24629
Modified:
lldb/trunk/packages/Python/lldbsuite/test/configuration.py
lldb/trunk/packages/Python/lldbsuite/test/dotest.py
lldb/trunk/packages/Python/lldbsuite/test/dotest_args.py
lldb/trunk/packages/Python/lldbsuite/test/test_result.py
Modified: lldb/trunk/packages/Python/lldbsuite/test/configuration.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/configuration.py?rev=282298&r1=282297&r2=282298&view=diff
==============================================================================
--- lldb/trunk/packages/Python/lldbsuite/test/configuration.py (original)
+++ lldb/trunk/packages/Python/lldbsuite/test/configuration.py Fri Sep 23 16:32:47 2016
@@ -101,6 +101,12 @@ parsable = False
# our test cases.
regexp = None
+# Sets of tests which are excluded at runtime
+skip_files = None
+skip_methods = None
+xfail_files = None
+xfail_methods = None
+
# By default, recorded session info for errored/failed test are dumped into its
# own file under a session directory named after the timestamp of the test suite
# run. Use '-s session-dir-name' to specify a specific dir name.
Modified: lldb/trunk/packages/Python/lldbsuite/test/dotest.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/dotest.py?rev=282298&r1=282297&r2=282298&view=diff
==============================================================================
--- lldb/trunk/packages/Python/lldbsuite/test/dotest.py (original)
+++ lldb/trunk/packages/Python/lldbsuite/test/dotest.py Fri Sep 23 16:32:47 2016
@@ -26,6 +26,7 @@ import atexit
import os
import errno
import platform
+import re
import signal
import socket
import subprocess
@@ -202,6 +203,48 @@ o GDB_REMOTE_LOG: if defined, specifies
sys.exit(0)
+def parseExclusion(exclusion_file):
+ """Parse an exclusion file, of the following format, where
+ 'skip files', 'skip methods', 'xfail files', and 'xfail methods'
+ are the possible list heading values:
+
+ skip files
+ <file name>
+ <file name>
+
+ xfail methods
+ <method name>
+ """
+ excl_type = None
+ case_type = None
+
+ with open(exclusion_file) as f:
+ for line in f:
+ if not excl_type:
+ [excl_type, case_type] = line.split()
+ continue
+
+ line = line.strip()
+ if not line:
+ excl_type = None
+ elif excl_type == 'skip' and case_type == 'files':
+ if not configuration.skip_files:
+ configuration.skip_files = []
+ configuration.skip_files.append(line)
+ elif excl_type == 'skip' and case_type == 'methods':
+ if not configuration.skip_methods:
+ configuration.skip_methods = []
+ configuration.skip_methods.append(line)
+ elif excl_type == 'xfail' and case_type == 'files':
+ if not configuration.xfail_files:
+ configuration.xfail_files = []
+ configuration.xfail_files.append(line)
+ elif excl_type == 'xfail' and case_type == 'methods':
+ if not configuration.xfail_methods:
+ configuration.xfail_methods = []
+ configuration.xfail_methods.append(line)
+
+
def parseOptionsAndInitTestdirs():
"""Initialize the list of directories containing our unittest scripts.
@@ -331,6 +374,9 @@ def parseOptionsAndInitTestdirs():
if args.executable:
lldbtest_config.lldbExec = os.path.realpath(args.executable)
+ if args.excluded:
+ parseExclusion(args.excluded)
+
if args.p:
if args.p.startswith('-'):
usage(parser)
@@ -749,11 +795,15 @@ def setupSysPath():
def visit_file(dir, name):
# Try to match the regexp pattern, if specified.
if configuration.regexp:
- import re
if not re.search(configuration.regexp, name):
# We didn't match the regex, we're done.
return
+ if configuration.skip_files:
+ for file_regexp in configuration.skip_files:
+ if re.search(file_regexp, name):
+ return
+
# We found a match for our test. Add it to the suite.
# Update the sys.path first.
Modified: lldb/trunk/packages/Python/lldbsuite/test/dotest_args.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/dotest_args.py?rev=282298&r1=282297&r2=282298&view=diff
==============================================================================
--- lldb/trunk/packages/Python/lldbsuite/test/dotest_args.py (original)
+++ lldb/trunk/packages/Python/lldbsuite/test/dotest_args.py Fri Sep 23 16:32:47 2016
@@ -96,6 +96,9 @@ def create_parser():
'-p',
metavar='pattern',
help='Specify a regexp filename pattern for inclusion in the test suite')
+ group.add_argument('--excluded', metavar='exclusion-file', help=textwrap.dedent(
+ '''Specify a file for tests to exclude. File should contain lists of regular expressions for test files or methods,
+ with each list under a matching header (xfail files, xfail methods, skip files, skip methods)'''))
group.add_argument(
'-G',
'--category',
Modified: lldb/trunk/packages/Python/lldbsuite/test/test_result.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/test_result.py?rev=282298&r1=282297&r2=282298&view=diff
==============================================================================
--- lldb/trunk/packages/Python/lldbsuite/test/test_result.py (original)
+++ lldb/trunk/packages/Python/lldbsuite/test/test_result.py Fri Sep 23 16:32:47 2016
@@ -18,6 +18,8 @@ import os
# Third-party modules
import unittest2
+from unittest2.util import strclass
+
# LLDB Modules
from . import configuration
from lldbsuite.test_event.event_builder import EventBuilder
@@ -124,10 +126,23 @@ class LLDBTestResult(unittest2.TextTestR
test,
test._testMethodName).__func__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run"
+ def checkExclusion(self, exclusion_list, name):
+ if exclusion_list:
+ import re
+ for item in exclusion_list:
+ if re.search(item, name):
+ return True
+ return False
+
def startTest(self, test):
if configuration.shouldSkipBecauseOfCategories(
self.getCategoriesForTest(test)):
self.hardMarkAsSkipped(test)
+ if self.checkExclusion(
+ configuration.skip_methods,
+ test._testMethodName):
+ self.hardMarkAsSkipped(test)
+
configuration.setCrashInfoHook(
"%s at %s" %
(str(test), inspect.getfile(
@@ -145,6 +160,15 @@ class LLDBTestResult(unittest2.TextTestR
EventBuilder.event_for_start(test))
def addSuccess(self, test):
+ if self.checkExclusion(
+ configuration.xfail_files,
+ strclass(
+ test.__class__)) or self.checkExclusion(
+ configuration.xfail_methods,
+ test._testMethodName):
+ self.addUnexpectedSuccess(test, None)
+ return
+
super(LLDBTestResult, self).addSuccess(test)
if configuration.parsable:
self.stream.write(
@@ -214,6 +238,15 @@ class LLDBTestResult(unittest2.TextTestR
test, err))
def addFailure(self, test, err):
+ if self.checkExclusion(
+ configuration.xfail_files,
+ strclass(
+ test.__class__)) or self.checkExclusion(
+ configuration.xfail_methods,
+ test._testMethodName):
+ self.addExpectedFailure(test, err, None)
+ return
+
configuration.sdir_has_content = True
super(LLDBTestResult, self).addFailure(test, err)
method = getattr(test, "markFailure", None)
More information about the lldb-commits
mailing list