[llvm-commits] [zorg] r172327 - in /zorg/trunk/zorg/buildbot: builders/ClangBuilder.py commands/LitTestCommand.py

[David Dean]

I'm seeing one more bit of fallout from this change.

When lnt tests fail, we don't have individual test logs, but we used to get a link on the build page with a list of the actually failing tests e.g. http://lab.llvm.org:8011/builders/clang-x86_64-darwin10-nt-O3-vectorize/builds/2250/steps/lnt.nightly-test/logs/tests.FAIL

I'm not seeing this on any of my masters any more.
Could you please investigate?

On 12 Jan 2013, at 12:50 PM, David Blaikie <dblaikie at gmail.com> wrote:

> Author: dblaikie
> Date: Sat Jan 12 14:50:32 2013
> New Revision: 172327
> 
> URL: http://llvm.org/viewvc/llvm-project?rev=172327&view=rev
> Log:
> Improve LitTestCommand
> 
> * use a LogObserver to provide dynamic results
> * display specific result types rather than warn/pass/fail
> 
> Modified:
>    zorg/trunk/zorg/buildbot/builders/ClangBuilder.py
>    zorg/trunk/zorg/buildbot/commands/LitTestCommand.py
> 
> Modified: zorg/trunk/zorg/buildbot/builders/ClangBuilder.py
> URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/ClangBuilder.py?rev=172327&r1=172326&r2=172327&view=diff
> ==============================================================================
> --- zorg/trunk/zorg/buildbot/builders/ClangBuilder.py (original)
> +++ zorg/trunk/zorg/buildbot/builders/ClangBuilder.py Sat Jan 12 14:50:32 2013
> @@ -211,7 +211,7 @@
>     if run_cxx_tests:
>         extraTestDirs += '%(builddir)s/llvm/tools/clang/utils/C++Tests'
>     if test:
> -        f.addStep(ClangTestCommand(name='check-all',
> +        f.addStep(LitTestCommand(name='check-all',
>                                    command=[make, "check-all", "VERBOSE=1",
>                                             WithProperties("LIT_ARGS=%s" % llvmTestArgs),
>                                             WithProperties("TESTARGS=%s" % clangTestArgs),
> 
> Modified: zorg/trunk/zorg/buildbot/commands/LitTestCommand.py
> URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/commands/LitTestCommand.py?rev=172327&r1=172326&r2=172327&view=diff
> ==============================================================================
> --- zorg/trunk/zorg/buildbot/commands/LitTestCommand.py (original)
> +++ zorg/trunk/zorg/buildbot/commands/LitTestCommand.py Sat Jan 12 14:50:32 2013
> @@ -1,50 +1,73 @@
> import re
> -import StandardizedTest
> -
> -class LitTestCommand(StandardizedTest.StandardizedTest):
> -  kTestLineRE = re.compile(r'([^ ]*): (.*) \(.*.*\)')
> -  kTestFailureLogStartRE = re.compile(r"""\*{4,80} TEST '(.*)' .*""")
> -  kTestFailureLogStopRE = re.compile(r"""\*{10,80}""")
> -
> -  def parseLog(self, lines):
> -    results = []
> -    results_by_name = {}
> -    failureLogs = []
> -    lines = self.getLog('stdio').readlines()
> -
> -    it = iter(lines)
> -    inFailure = None
> -    for ln in it:
> +import urllib
> +import buildbot
> +import buildbot.status.builder
> +from buildbot.status.results import FAILURE, SUCCESS
> +import buildbot.steps.shell
> +from buildbot.process.buildstep import LogLineObserver
> +from buildbot.steps.shell import Test
> +
> +class LitLogObserver(LogLineObserver):
> +    kTestLineRE = re.compile(r'([^ ]*): (.*) \(.*.*\)')
> +    kTestFailureLogStartRE = re.compile(r"""\*{4,80} TEST '(.*)' .*""")
> +    kTestFailureLogStopRE = re.compile(r"""\*{10,80}""")
> +    def __init__(self):
> +        LogLineObserver.__init__(self)
> +        self.resultCounts = {}
> +        self.inFailure = None
> +    def outLineReceived(self, line):
>       # See if we are inside a failure log.
> -      if inFailure:
> -        inFailure[1].append(ln)
> -        if self.kTestFailureLogStopRE.match(ln):
> -          name,log = inFailure
> -          if name not in results_by_name:
> -            raise ValueError,'Invalid log result with no status line!'
> -          results_by_name[name][2] = ''.join(log) + '\n'
> -          inFailure = None
> -        continue
> -
> -      ln = ln.strip()
> -      if not ln:
> -        continue
> +      if self.inFailure:
> +        self.inFailure[1].append(line)
> +        if self.kTestFailureLogStopRE.match(line):
> +          name,log = self.inFailure
> +          self.step.addCompleteLog(name.replace('/', '__'), '\n'.join(log))
> +          self.inFailure = None
> +        return
> +
> +      line = line.strip()
> +      if not line:
> +        return
> 
>       # Check for test failure logs.
> -      m = self.kTestFailureLogStartRE.match(ln)
> +      m = self.kTestFailureLogStartRE.match(line)
>       if m:
> -        inFailure = (m.group(1), [ln])
> -        continue
> +        self.inFailure = (m.group(1), [line])
> +        return
> 
>       # Otherwise expect a test status line.
> -      m = self.kTestLineRE.match(ln)
> +      m = self.kTestLineRE.match(line)
>       if m:
> -        code, name = m.group(1),m.group(2)
> -        results.append( [code, name, None] )
> -        results_by_name[name] = results[-1]
> -
> -    if inFailure:
> -      raise ValueError,("Unexpected clang test running output, "
> -                        "unterminated failure log!")
> -
> -    return results
> +        code, name = m.groups()
> +        if not code in self.resultCounts:
> +          self.resultCounts[code] = 0
> +        self.resultCounts[code] += 1
> +
> +class LitTestCommand(Test):
> +    resultNames = {'FAIL':'unexpected failures',
> +                   'PASS':'expected passes',
> +                   'XFAIL':'expected failures',
> +                   'XPASS':'unexpected passes',
> +                   'KFAIL':'known failures',
> +                   'KPASS':'unknown passes',
> +                   'UNRESOLVED':'unresolved testcases',
> +                   'UNTESTED':'untested testcases',
> +                   'UNSUPPORTED':'unsupported tests'}
> +    failingCodes = set(['FAIL', 'XPASS', 'KPASS', 'UNRESOLVED'])
> +
> +    def __init__(self, ignore=[], flaky=[], max_logs=20,
> +                 *args, **kwargs):
> +        Test.__init__(self, *args, **kwargs)
> +        self.logObserver = LitLogObserver()
> +        self.addLogObserver('stdio', self.logObserver)
> +
> +    def evaluateCommand(self, cmd):
> +        if any([r in self.logObserver.resultCounts for r in self.failingCodes]):
> +            return FAILURE
> +        return SUCCESS
> +
> +    def describe(self, done=False):
> +        description = Test.describe(self, done)
> +        for name, count in self.logObserver.resultCounts.iteritems():
> +            description.append('{0} {1}'.format(count, self.resultNames[name]))
> +        return description
> 
> 
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at cs.uiuc.edu
> http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits

-David