[compiler-rt] dadc214 - Title: [TSAN] Parameterize the hard-coded threshold of deflake in tsan test

[Anh Tuyen Tran via llvm-commits]

Author: Anh Tuyen Tran
Date: 2020-02-12T15:51:57Z
New Revision: dadc214e4d9d09a8a7a9f15780c1201c18f93e05

URL: https://github.com/llvm/llvm-project/commit/dadc214e4d9d09a8a7a9f15780c1201c18f93e05
DIFF: https://github.com/llvm/llvm-project/commit/dadc214e4d9d09a8a7a9f15780c1201c18f93e05.diff

LOG: Title: [TSAN] Parameterize the hard-coded threshold of deflake in tsan test

Summary:
A number of testcases in TSAN are designed to deal with intermittent problems
not exist in all executions of the tested program. A script called deflake.bash
runs the executable up to 10 times to deal with the intermittent nature of the tests.

The purpose of this patch is to parameterize the hard-coded threshold above via
--cmake_variables=-DTSAN_TEST_DEFLAKE_THRESHOLD=SomeIntegerValue
When this cmake var is not set, the default value of 10 will be used.

Reviewer: dvyukov (Dmitry Vyukov), eugenis (Evgenii Stepanov), rnk (Reid Kleckner), hubert.reinterpretcast (Hubert Tong), vitalybuka (Vitaly Buka)

Reviewed By: vitalybuka (Vitaly Buka)

Subscribers: mgorny (Michal Gorny), jfb (JF Bastien), steven.zhang (qshanz), llvm-commits (Mailing List llvm-commits), Sanitizers

Tag: LLVM, Sanitizers

Differential Revision: https://reviews.llvm.org/D73707

Added: 
    

Modified: 
    compiler-rt/test/tsan/CMakeLists.txt
    compiler-rt/test/tsan/deflake.bash
    compiler-rt/test/tsan/lit.cfg.py
    compiler-rt/test/tsan/lit.site.cfg.py.in

Removed: 
    


################################################################################
diff  --git a/compiler-rt/test/tsan/CMakeLists.txt b/compiler-rt/test/tsan/CMakeLists.txt
index 35b0f417a6c6..7cc3537660a6 100644
--- a/compiler-rt/test/tsan/CMakeLists.txt
+++ b/compiler-rt/test/tsan/CMakeLists.txt
@@ -19,6 +19,10 @@ endif()
 
 set(TSAN_TESTSUITES)
 
+if (NOT DEFINED TSAN_TEST_DEFLAKE_THRESHOLD)
+  set(TSAN_TEST_DEFLAKE_THRESHOLD "10")
+endif()
+
 set(TSAN_TEST_ARCH ${TSAN_SUPPORTED_ARCH})
 if(APPLE)
   darwin_filter_host_archs(TSAN_SUPPORTED_ARCH TSAN_TEST_ARCH)

diff  --git a/compiler-rt/test/tsan/deflake.bash b/compiler-rt/test/tsan/deflake.bash
index 9731fa53e589..5a88174a7ce1 100755
--- a/compiler-rt/test/tsan/deflake.bash
+++ b/compiler-rt/test/tsan/deflake.bash
@@ -1,13 +1,22 @@
 #!/usr/bin/env bash
 # This script is used to deflake inherently flaky tsan tests.
 # It is invoked from lit tests as:
-# %deflake mybinary
+# %deflake $THRESHOLD  mybinary
 # which is then substituted by lit to:
-# $(dirname %s)/deflake.bash mybinary
-# The script runs the target program up to 10 times,
+# $(dirname %s)/deflake.bash $THRESHOLD mybinary
+# - When TSAN_TEST_DEFLAKE_THRESHOLD is defined to a positive integer value,
+#   THRESHOLD will be the defined value.
+# - When TSAN_TEST_DEFLAKE_THRESHOLD is not defined, THRESHOLD will be 10.
+# The script runs the target program up to $THRESHOLD times,
 # until it fails (i.e. produces a race report).
 
-for i in $(seq 1 10); do
+THRESHOLD="${1}"
+shift
+
+# Early exit if $THRESHOLD is not a non-negative integer
+[[  "${THRESHOLD}" =~ ^[0-9]+$ ]] || exit 1
+
+while (( THRESHOLD-- )); do
 	OUT=`$@ 2>&1`
 	if [[ $? != 0 ]]; then
 		echo "$OUT"

diff  --git a/compiler-rt/test/tsan/lit.cfg.py b/compiler-rt/test/tsan/lit.cfg.py
index ece8e66a2209..a07adaa0d657 100644
--- a/compiler-rt/test/tsan/lit.cfg.py
+++ b/compiler-rt/test/tsan/lit.cfg.py
@@ -75,7 +75,7 @@ def build_invocation(compile_flags):
 # Define CHECK-%os to check for OS-dependent output.
 config.substitutions.append( ('CHECK-%os', ("CHECK-" + config.host_os)))
 
-config.substitutions.append( ("%deflake ", os.path.join(os.path.dirname(__file__), "deflake.bash") + " "))
+config.substitutions.append( ("%deflake ", os.path.join(os.path.dirname(__file__), "deflake.bash") + " " + config.deflake_threshold))
 
 # Default test suffixes.
 config.suffixes = ['.c', '.cpp', '.m', '.mm']

diff  --git a/compiler-rt/test/tsan/lit.site.cfg.py.in b/compiler-rt/test/tsan/lit.site.cfg.py.in
index a85025980c2b..5e8d610e5df0 100644
--- a/compiler-rt/test/tsan/lit.site.cfg.py.in
+++ b/compiler-rt/test/tsan/lit.site.cfg.py.in
@@ -6,6 +6,7 @@ config.has_libcxx = @TSAN_HAS_LIBCXX@
 config.apple_platform = "@TSAN_TEST_APPLE_PLATFORM@"
 config.target_cflags = "@TSAN_TEST_TARGET_CFLAGS@"
 config.target_arch = "@TSAN_TEST_TARGET_ARCH@"
+config.deflake_threshold = "@TSAN_TEST_DEFLAKE_THRESHOLD@"
 
 # Load common config for all compiler-rt lit tests.
 lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured")