[llvm] r332987 - [lit] Try to make `shtest-timeout.py` test more reliable by using a

[Dan Liew via llvm-commits]

Author: delcypher
Date: Tue May 22 08:06:29 2018
New Revision: 332987

URL: http://llvm.org/viewvc/llvm-project?rev=332987&view=rev
Log:
[lit] Try to make `shtest-timeout.py` test more reliable by using a
larger timeout value. This really isn't very good because it will
still be susceptible to machine performance.

While we are here also fix a bug in validation of
`maxIndividualTestTime` where previously it wasn't checked if the
type was an int.

rdar://problem/40221572

Modified:
    llvm/trunk/utils/lit/lit/LitConfig.py
    llvm/trunk/utils/lit/tests/Inputs/shtest-timeout/lit.cfg
    llvm/trunk/utils/lit/tests/shtest-timeout.py

Modified: llvm/trunk/utils/lit/lit/LitConfig.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/LitConfig.py?rev=332987&r1=332986&r2=332987&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/LitConfig.py (original)
+++ llvm/trunk/utils/lit/lit/LitConfig.py Tue May 22 08:06:29 2018
@@ -83,6 +83,8 @@ class LitConfig(object):
             Interface for setting maximum time to spend executing
             a single test
         """
+        if not isinstance(value, int):
+            self.fatal('maxIndividualTestTime must set to a value of type int.')
         self._maxIndividualTestTime = value
         if self.maxIndividualTestTime > 0:
             # The current implementation needs psutil to set

Modified: llvm/trunk/utils/lit/tests/Inputs/shtest-timeout/lit.cfg
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/tests/Inputs/shtest-timeout/lit.cfg?rev=332987&r1=332986&r2=332987&view=diff
==============================================================================
--- llvm/trunk/utils/lit/tests/Inputs/shtest-timeout/lit.cfg (original)
+++ llvm/trunk/utils/lit/tests/Inputs/shtest-timeout/lit.cfg Tue May 22 08:06:29 2018
@@ -17,9 +17,9 @@ else:
 
 configSetTimeout = lit_config.params.get('set_timeout', '0')
 
-if configSetTimeout == '1':
+if configSetTimeout != '0':
     # Try setting the max individual test time in the configuration
-    lit_config.maxIndividualTestTime = 1
+    lit_config.maxIndividualTestTime = int(configSetTimeout)
 
 config.test_format = lit.formats.ShTest(execute_external=externalShell)
 config.suffixes = ['.py']

Modified: llvm/trunk/utils/lit/tests/shtest-timeout.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/tests/shtest-timeout.py?rev=332987&r1=332986&r2=332987&view=diff
==============================================================================
--- llvm/trunk/utils/lit/tests/shtest-timeout.py (original)
+++ llvm/trunk/utils/lit/tests/shtest-timeout.py Tue May 22 08:06:29 2018
@@ -3,11 +3,16 @@
 # PR33944
 # XFAIL: windows
 
+# FIXME: This test is fragile because it relies on time which can
+# be affected by system performance. In particular we are currently
+# assuming that `short.py` can be successfully executed within 2
+# seconds of wallclock time.
+
 # Test per test timeout using external shell
 # RUN: not %{lit} \
 # RUN: %{inputs}/shtest-timeout/infinite_loop.py \
 # RUN: %{inputs}/shtest-timeout/short.py \
-# RUN: -j 1 -v --debug --timeout 1 --param external=1 > %t.extsh.out 2> %t.extsh.err
+# RUN: -j 1 -v --debug --timeout 2 --param external=1 > %t.extsh.out 2> %t.extsh.err
 # RUN: FileCheck --check-prefix=CHECK-OUT-COMMON < %t.extsh.out %s
 # RUN: FileCheck --check-prefix=CHECK-EXTSH-ERR < %t.extsh.err %s
 #
@@ -17,7 +22,7 @@
 # RUN: not %{lit} \
 # RUN: %{inputs}/shtest-timeout/infinite_loop.py \
 # RUN: %{inputs}/shtest-timeout/short.py \
-# RUN: -j 1 -v --debug --timeout 1 --param external=0 > %t.intsh.out 2> %t.intsh.err
+# RUN: -j 1 -v --debug --timeout 2 --param external=0 > %t.intsh.out 2> %t.intsh.err
 # RUN: FileCheck  --check-prefix=CHECK-OUT-COMMON < %t.intsh.out %s
 # RUN: FileCheck --check-prefix=CHECK-INTSH-OUT < %t.intsh.out %s
 # RUN: FileCheck --check-prefix=CHECK-INTSH-ERR < %t.intsh.err %s
@@ -33,14 +38,14 @@
 # RUN: %{inputs}/shtest-timeout/infinite_loop.py \
 # RUN: %{inputs}/shtest-timeout/short.py \
 # RUN: -j 1 -v --debug --param external=0 \
-# RUN: --param set_timeout=1 > %t.cfgset.out 2> %t.cfgset.err
+# RUN: --param set_timeout=2 > %t.cfgset.out 2> %t.cfgset.err
 # RUN: FileCheck --check-prefix=CHECK-OUT-COMMON  < %t.cfgset.out %s
 # RUN: FileCheck --check-prefix=CHECK-CFGSET-ERR < %t.cfgset.err %s
 #
 # CHECK-CFGSET-ERR: Using internal shell
 
 # CHECK-OUT-COMMON: TIMEOUT: per_test_timeout :: infinite_loop.py
-# CHECK-OUT-COMMON: Timeout: Reached timeout of 1 seconds
+# CHECK-OUT-COMMON: Timeout: Reached timeout of 2 seconds
 # CHECK-OUT-COMMON: Command {{([0-9]+ )?}}Output
 
 # CHECK-OUT-COMMON: PASS: per_test_timeout :: short.py