[libcxx] r280050 - Add "FLAKY_TEST" test directive to support re-running flaky tests.
Eric Fiselier via cfe-commits
cfe-commits at lists.llvm.org
Mon Aug 29 18:46:44 PDT 2016
Author: ericwf
Date: Mon Aug 29 20:46:43 2016
New Revision: 280050
URL: http://llvm.org/viewvc/llvm-project?rev=280050&view=rev
Log:
Add "FLAKY_TEST" test directive to support re-running flaky tests.
Some of the mutex tests fail on machines with high load. This patch implements
the test directive "// FLAKY_TEST" which allows a test to be run 3 times
before it's considered a failure.
Modified:
libcxx/trunk/test/libcxx/test/format.py
libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex.pass.cpp
libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_duration.pass.cpp
libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_time_point.pass.cpp
libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_duration.pass.cpp
libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_time_point.pass.cpp
libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock.pass.cpp
libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock_shared.pass.cpp
libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/try_lock_shared.pass.cpp
libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock.pass.cpp
libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock_shared.pass.cpp
libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_for.pass.cpp
libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared.pass.cpp
libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_for.pass.cpp
libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_until.pass.cpp
libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_until.pass.cpp
Modified: libcxx/trunk/test/libcxx/test/format.py
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/libcxx/test/format.py?rev=280050&r1=280049&r2=280050&view=diff
==============================================================================
--- libcxx/trunk/test/libcxx/test/format.py (original)
+++ libcxx/trunk/test/libcxx/test/format.py Mon Aug 29 20:46:43 2016
@@ -114,6 +114,9 @@ class LibcxxTestFormat(object):
def _evaluate_pass_test(self, test, tmpBase, lit_config):
execDir = os.path.dirname(test.getExecPath())
source_path = test.getSourcePath()
+ with open(source_path, 'r') as f:
+ contents = f.read()
+ is_flaky = 'FLAKY_TEST' in contents
exec_path = tmpBase + '.exe'
object_path = tmpBase + '.o'
# Create the output directory if it does not already exist.
@@ -139,14 +142,21 @@ class LibcxxTestFormat(object):
# should add a `// FILE-DEP: foo.dat` to each test to track this.
data_files = [os.path.join(local_cwd, f)
for f in os.listdir(local_cwd) if f.endswith('.dat')]
- cmd, out, err, rc = self.executor.run(exec_path, [exec_path],
- local_cwd, data_files, env)
- if rc != 0:
- report = libcxx.util.makeReport(cmd, out, err, rc)
- report = "Compiled With: %s\n%s" % (compile_cmd, report)
- report += "Compiled test failed unexpectedly!"
- return lit.Test.FAIL, report
- return lit.Test.PASS, ''
+ max_retry = 3 if is_flaky else 1
+ for retry_count in range(max_retry):
+ cmd, out, err, rc = self.executor.run(exec_path, [exec_path],
+ local_cwd, data_files,
+ env)
+ if rc == 0:
+ res = lit.Test.PASS if retry_count == 0 else lit.Test.FLAKYPASS
+ return res, ''
+ elif rc != 0 and retry_count + 1 == max_retry:
+ report = libcxx.util.makeReport(cmd, out, err, rc)
+ report = "Compiled With: %s\n%s" % (compile_cmd, report)
+ report += "Compiled test failed unexpectedly!"
+ return lit.Test.FAIL, report
+
+ assert False # Unreachable
finally:
# Note that cleanup of exec_file happens in `_clean()`. If you
# override this, cleanup is your reponsibility.
Modified: libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex.pass.cpp
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex.pass.cpp?rev=280050&r1=280049&r2=280050&view=diff
==============================================================================
--- libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex.pass.cpp (original)
+++ libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex.pass.cpp Mon Aug 29 20:46:43 2016
@@ -10,6 +10,8 @@
// UNSUPPORTED: libcpp-has-no-threads
// UNSUPPORTED: c++98, c++03, c++11
+// FLAKY_TEST
+
// <shared_mutex>
// template <class Mutex> class shared_lock;
Modified: libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_duration.pass.cpp
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_duration.pass.cpp?rev=280050&r1=280049&r2=280050&view=diff
==============================================================================
--- libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_duration.pass.cpp (original)
+++ libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_duration.pass.cpp Mon Aug 29 20:46:43 2016
@@ -10,6 +10,8 @@
// UNSUPPORTED: libcpp-has-no-threads
// UNSUPPORTED: c++98, c++03, c++11
+// FLAKY_TEST
+
// <shared_mutex>
// class timed_mutex;
Modified: libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_time_point.pass.cpp
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_time_point.pass.cpp?rev=280050&r1=280049&r2=280050&view=diff
==============================================================================
--- libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_time_point.pass.cpp (original)
+++ libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_time_point.pass.cpp Mon Aug 29 20:46:43 2016
@@ -10,6 +10,8 @@
// UNSUPPORTED: libcpp-has-no-threads
// UNSUPPORTED: c++98, c++03, c++11
+// FLAKY_TEST
+
// <shared_mutex>
// class shared_timed_mutex;
Modified: libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_duration.pass.cpp
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_duration.pass.cpp?rev=280050&r1=280049&r2=280050&view=diff
==============================================================================
--- libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_duration.pass.cpp (original)
+++ libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_duration.pass.cpp Mon Aug 29 20:46:43 2016
@@ -9,6 +9,8 @@
//
// UNSUPPORTED: libcpp-has-no-threads
+// FLAKY_TEST
+
// <mutex>
// class timed_mutex;
Modified: libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_time_point.pass.cpp
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_time_point.pass.cpp?rev=280050&r1=280049&r2=280050&view=diff
==============================================================================
--- libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_time_point.pass.cpp (original)
+++ libcxx/trunk/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_time_point.pass.cpp Mon Aug 29 20:46:43 2016
@@ -9,6 +9,8 @@
//
// UNSUPPORTED: libcpp-has-no-threads
+// FLAKY_TEST
+
// <mutex>
// class timed_mutex;
Modified: libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock.pass.cpp
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock.pass.cpp?rev=280050&r1=280049&r2=280050&view=diff
==============================================================================
--- libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock.pass.cpp (original)
+++ libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock.pass.cpp Mon Aug 29 20:46:43 2016
@@ -10,6 +10,8 @@
// UNSUPPORTED: libcpp-has-no-threads
// UNSUPPORTED: c++03, c++98, c++11, c++14
+// FLAKY_TEST
+
// <shared_mutex>
// class shared_mutex;
Modified: libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock_shared.pass.cpp
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock_shared.pass.cpp?rev=280050&r1=280049&r2=280050&view=diff
==============================================================================
--- libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock_shared.pass.cpp (original)
+++ libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock_shared.pass.cpp Mon Aug 29 20:46:43 2016
@@ -10,6 +10,8 @@
// UNSUPPORTED: libcpp-has-no-threads
// UNSUPPORTED: c++03, c++98, c++11, c++14
+// FLAKY_TEST
+
// <shared_mutex>
// class shared_mutex;
Modified: libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/try_lock_shared.pass.cpp
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/try_lock_shared.pass.cpp?rev=280050&r1=280049&r2=280050&view=diff
==============================================================================
--- libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/try_lock_shared.pass.cpp (original)
+++ libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/try_lock_shared.pass.cpp Mon Aug 29 20:46:43 2016
@@ -10,6 +10,8 @@
// UNSUPPORTED: libcpp-has-no-threads
// UNSUPPORTED: c++03, c++98, c++11, c++14
+// FLAKY_TEST
+
// <shared_mutex>
// class shared_mutex;
Modified: libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock.pass.cpp
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock.pass.cpp?rev=280050&r1=280049&r2=280050&view=diff
==============================================================================
--- libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock.pass.cpp (original)
+++ libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock.pass.cpp Mon Aug 29 20:46:43 2016
@@ -10,6 +10,8 @@
// UNSUPPORTED: libcpp-has-no-threads
// UNSUPPORTED: c++03, c++98, c++11
+// FLAKY_TEST
+
// <shared_mutex>
// class shared_timed_mutex;
Modified: libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock_shared.pass.cpp
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock_shared.pass.cpp?rev=280050&r1=280049&r2=280050&view=diff
==============================================================================
--- libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock_shared.pass.cpp (original)
+++ libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock_shared.pass.cpp Mon Aug 29 20:46:43 2016
@@ -10,6 +10,8 @@
// UNSUPPORTED: libcpp-has-no-threads
// UNSUPPORTED: c++03, c++98, c++11
+// FLAKY_TEST
+
// <shared_mutex>
// class shared_timed_mutex;
Modified: libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_for.pass.cpp
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_for.pass.cpp?rev=280050&r1=280049&r2=280050&view=diff
==============================================================================
--- libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_for.pass.cpp (original)
+++ libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_for.pass.cpp Mon Aug 29 20:46:43 2016
@@ -10,6 +10,8 @@
// UNSUPPORTED: libcpp-has-no-threads
// UNSUPPORTED: c++03, c++98, c++11
+// FLAKY_TEST
+
// <shared_mutex>
// class shared_timed_mutex;
Modified: libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared.pass.cpp
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared.pass.cpp?rev=280050&r1=280049&r2=280050&view=diff
==============================================================================
--- libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared.pass.cpp (original)
+++ libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared.pass.cpp Mon Aug 29 20:46:43 2016
@@ -10,6 +10,8 @@
// UNSUPPORTED: libcpp-has-no-threads
// UNSUPPORTED: c++03, c++98, c++11
+// FLAKY_TEST
+
// <shared_mutex>
// class shared_timed_mutex;
Modified: libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_for.pass.cpp
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_for.pass.cpp?rev=280050&r1=280049&r2=280050&view=diff
==============================================================================
--- libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_for.pass.cpp (original)
+++ libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_for.pass.cpp Mon Aug 29 20:46:43 2016
@@ -10,6 +10,8 @@
// UNSUPPORTED: libcpp-has-no-threads
// UNSUPPORTED: c++03, c++98, c++11
+// FLAKY_TEST
+
// <shared_mutex>
// class shared_timed_mutex;
Modified: libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_until.pass.cpp
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_until.pass.cpp?rev=280050&r1=280049&r2=280050&view=diff
==============================================================================
--- libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_until.pass.cpp (original)
+++ libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_until.pass.cpp Mon Aug 29 20:46:43 2016
@@ -10,6 +10,8 @@
// UNSUPPORTED: libcpp-has-no-threads
// UNSUPPORTED: c++03, c++98, c++11
+// FLAKY_TEST
+
// <shared_mutex>
// class shared_timed_mutex;
Modified: libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_until.pass.cpp
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_until.pass.cpp?rev=280050&r1=280049&r2=280050&view=diff
==============================================================================
--- libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_until.pass.cpp (original)
+++ libcxx/trunk/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_until.pass.cpp Mon Aug 29 20:46:43 2016
@@ -10,6 +10,8 @@
// UNSUPPORTED: libcpp-has-no-threads
// UNSUPPORTED: c++03, c++98, c++11
+// FLAKY_TEST
+
// <shared_mutex>
// class shared_timed_mutex;
More information about the cfe-commits
mailing list