[llvm-commits] [llvm] r88827 - in /llvm/trunk: docs/CommandGuide/lit.pod utils/lit/Test.py utils/lit/TestRunner.py utils/lit/lit.py

[Daniel Dunbar]

Author: ddunbar
Date: Sat Nov 14 19:02:09 2009
New Revision: 88827

URL: http://llvm.org/viewvc/llvm-project?rev=88827&view=rev
Log:
lit: Add --repeat=N option, for running each test N times.
 - Currently just useful for timing, although it could be extended as one (bad) way to deal with flaky tests.

Modified:
    llvm/trunk/docs/CommandGuide/lit.pod
    llvm/trunk/utils/lit/Test.py
    llvm/trunk/utils/lit/TestRunner.py
    llvm/trunk/utils/lit/lit.py

Modified: llvm/trunk/docs/CommandGuide/lit.pod
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/docs/CommandGuide/lit.pod?rev=88827&r1=88826&r2=88827&view=diff

==============================================================================
--- llvm/trunk/docs/CommandGuide/lit.pod (original)
+++ llvm/trunk/docs/CommandGuide/lit.pod Sat Nov 14 19:02:09 2009
@@ -149,6 +149,11 @@
 
 Run Tcl scripts internally (instead of converting to shell scripts).
 
+=item B<--repeat>=I<N>
+
+Run each test I<N> times. Currently this is primarily useful for timing tests,
+other results are not collated in any reasonable fashion.
+
 =back
 
 =head1 EXIT STATUS

Modified: llvm/trunk/utils/lit/Test.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/Test.py?rev=88827&r1=88826&r2=88827&view=diff

==============================================================================
--- llvm/trunk/utils/lit/Test.py (original)
+++ llvm/trunk/utils/lit/Test.py Sat Nov 14 19:02:09 2009
@@ -54,6 +54,14 @@
         self.output = None
         # The wall time to execute this test, if timing and once complete.
         self.elapsed = None
+        # The repeat index of this test, or None.
+        self.index = None
+
+    def copyWithIndex(self, index):
+        import copy
+        res = copy.copy(self)
+        res.index = index
+        return res
 
     def setResult(self, result, output, elapsed):
         assert self.result is None, "Test result already set!"

Modified: llvm/trunk/utils/lit/TestRunner.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/TestRunner.py?rev=88827&r1=88826&r2=88827&view=diff

==============================================================================
--- llvm/trunk/utils/lit/TestRunner.py (original)
+++ llvm/trunk/utils/lit/TestRunner.py Sat Nov 14 19:02:09 2009
@@ -367,6 +367,8 @@
     execpath = test.getExecPath()
     execdir,execbase = os.path.split(execpath)
     tmpBase = os.path.join(execdir, 'Output', execbase)
+    if test.index is not None:
+        tmpBase += '_%d' % test.index
 
     # We use #_MARKER_# to hide %% while we do the other substitutions.
     substitutions = [('%%', '#_MARKER_#')]

Modified: llvm/trunk/utils/lit/lit.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit.py?rev=88827&r1=88826&r2=88827&view=diff

==============================================================================
--- llvm/trunk/utils/lit/lit.py (original)
+++ llvm/trunk/utils/lit/lit.py Sat Nov 14 19:02:09 2009
@@ -388,6 +388,9 @@
     group.add_option("", "--no-tcl-as-sh", dest="useTclAsSh",
                       help="Don't run Tcl scripts using 'sh'",
                       action="store_false", default=True)
+    group.add_option("", "--repeat", dest="repeatTests", metavar="N",
+                      help="Repeat tests N times (for timing)",
+                      action="store", default=None, type=int)
     parser.add_option_group(group)
 
     (opts, args) = parser.parse_args()
@@ -472,6 +475,11 @@
     header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
                                                       opts.numThreads)
 
+    if opts.repeatTests:
+        tests = [t.copyWithIndex(i)
+                 for t in tests
+                 for i in range(opts.repeatTests)]
+
     progressBar = None
     if not opts.quiet:
         if opts.succinct and opts.useProgressBar:
@@ -524,11 +532,16 @@
         print
 
     if opts.timeTests:
-        byTime = list(tests)
-        byTime.sort(key = lambda t: t.elapsed)
+        # Collate, in case we repeated tests.
+        times = {}
+        for t in tests:
+            key = t.getFullName()
+            times[key] = times.get(key, 0.) + t.elapsed
+
+        byTime = list(times.items())
+        byTime.sort(key = lambda (name,elapsed): elapsed)
         if byTime:
-            Util.printHistogram([(t.getFullName(), t.elapsed) for t in byTime],
-                                title='Tests')
+            Util.printHistogram(byTime, title='Tests')
 
     for name,code in (('Expected Passes    ', Test.PASS),
                       ('Expected Failures  ', Test.XFAIL),