[llvm-commits] [zorg] r101577 - in /zorg/trunk/zorg/buildbot/commands: LitTestCommand.py StandardizedTest.py

[Daniel Dunbar]

Author: ddunbar
Date: Fri Apr 16 18:36:10 2010
New Revision: 101577

URL: http://llvm.org/viewvc/llvm-project?rev=101577&view=rev
Log:
buildbot: Add a new StandardizedTest test command, which serves as a base class
for test steps that want to parse log files and bin the results into things like
PASS, FAIL, etc.

Also, add a new LitTestCommand based on this.

Added:
    zorg/trunk/zorg/buildbot/commands/LitTestCommand.py
    zorg/trunk/zorg/buildbot/commands/StandardizedTest.py

Added: zorg/trunk/zorg/buildbot/commands/LitTestCommand.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/commands/LitTestCommand.py?rev=101577&view=auto
==============================================================================
--- zorg/trunk/zorg/buildbot/commands/LitTestCommand.py (added)
+++ zorg/trunk/zorg/buildbot/commands/LitTestCommand.py Fri Apr 16 18:36:10 2010
@@ -0,0 +1,50 @@
+import re
+import StandardizedTest
+
+class LitTestCommand(StandardizedTest.StandardizedTest):
+  kTestLineRE = re.compile(r'([^ ]*): (.*) \(.*.*\)')
+  kTestFailureLogStartRE = re.compile(r"""\*{4,80} TEST '(.*)' .*""")
+  kTestFailureLogStopRE = re.compile(r"""\*{10,80}""")
+
+  def parseLog(self, lines):
+    results = []
+    results_by_name = {}
+    failureLogs = []
+    lines = self.getLog('stdio').readlines()
+
+    it = iter(lines)
+    inFailure = None
+    for ln in it:
+      # See if we are inside a failure log.
+      if inFailure:
+        inFailure[1].append(ln)
+        if self.kTestFailureLogStopRE.match(ln):
+          name,log = inFailure
+          if name not in results_by_name:
+            raise ValueError,'Invalid log result with no status line!'
+          results_by_name[name][2] = ''.join(log) + '\n'
+          inFailure = None
+        continue
+
+      ln = ln.strip()
+      if not ln:
+        continue
+
+      # Check for test failure logs.
+      m = self.kTestFailureLogStartRE.match(ln)
+      if m:
+        inFailure = (m.group(1), [ln])
+        continue
+
+      # Otherwise expect a test status line.
+      m = self.kTestLineRE.match(ln)
+      if m:
+        code, name = m.group(1),m.group(2)
+        results.append( [code, name, None] )
+        results_by_name[name] = results[-1]
+
+    if inFailure:
+      raise ValueError,("Unexpected clang test running output, "
+                        "unterminated failure log!")
+
+    return results

Added: zorg/trunk/zorg/buildbot/commands/StandardizedTest.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/commands/StandardizedTest.py?rev=101577&view=auto
==============================================================================
--- zorg/trunk/zorg/buildbot/commands/StandardizedTest.py (added)
+++ zorg/trunk/zorg/buildbot/commands/StandardizedTest.py Fri Apr 16 18:36:10 2010
@@ -0,0 +1,95 @@
+import re
+import buildbot
+import buildbot.status.builder
+import buildbot.steps.shell
+
+class StandardizedTest(buildbot.steps.shell.Test):
+    # FIXME: We should process things in a test observer instead of at the end.
+
+    knownCodes = ['FAIL', 'XFAIL', 'PASS', 'XPASS',
+                  'UNRESOLVED', 'UNSUPPORTED']
+    failingCodes = set(['FAIL', 'XPASS', 'UNRESOLVED'])
+    warningCodes = set(['IGN PASS', 'IGN XFAIL'])
+
+    # The list of all possible codes, including ignored codes. This is the
+    # display order, as well.
+    allKnownCodes = knownCodes + ['IGN ' + c for c in knownCodes]
+
+    testLogName = 'stdio'
+
+    def __init__(self, ignore=[], max_logs=20,
+                 *args, **kwargs):
+        buildbot.steps.shell.Test.__init__(self, *args, **kwargs)
+
+        self.ignoredTests = set(ignore)
+        self.maxLogs = int(max_logs)
+        self.addFactoryArguments(ignore=list(ignore))
+        self.addFactoryArguments(max_logs=max_logs)
+
+    def parseLog(self, log_lines):
+        """parseLog(log_lines) -> [(result_code, test_name, test_log), ...]"""
+        abstract
+
+    def evaluateCommand(self, cmd):
+        # Always fail if the command itself failed.
+        if cmd.rc != 0:
+            return buildbot.status.builder.FAILURE
+
+        results_by_code = {}
+        logs = []
+        lines = self.getLog(self.testLogName).readlines()
+        for result,test,log in self.parseLog(lines):
+            if result not in self.knownCodes:
+                raise ValueError,'test command return invalid result code!'
+
+            # Convert externally expected failures.
+            if test in self.ignoredTests:
+                result = 'IGN ' + result
+
+            if result not in results_by_code:
+                results_by_code[result] = set()
+
+            results_by_code[result].add(test)
+
+            # Add logs for failures.
+            if result in self.failingCodes and len(logs) < self.maxLogs:
+                if log is not None and log.strip():
+                    logs.append((test, log))
+
+        # Summarize result counts.
+        total = failed = passed = warnings = 0
+        for code in self.allKnownCodes:
+            results = results_by_code.get(code)
+            if not results:
+                continue
+
+            total += len(results)
+            if code in self.failingCodes:
+                failed += len(results)
+            elif code in self.warningCodes:
+                warnings += len(results)
+            else:
+                passed += len(results)
+
+            # Add a list of the tests in each category, for everything except
+            # PASS.
+            if code != 'PASS':
+                results = list(results)
+                results.sort()
+                self.addCompleteLog('tests.%s' % code,
+                                    '\n'.join(results) + '\n')
+
+        self.setTestResults(total=total, failed=failed,
+                            passed=passed, warnings=warnings)
+
+        # Add the logs.
+        logs.sort()
+        for test, log in logs:
+            self.addCompleteLog(test, log)
+
+        if failed:
+            return buildbot.status.builder.FAILURE
+        if warnings:
+            return buildbot.status.builder.WARNINGS
+
+        return buildbot.status.builder.SUCCESS