[PATCH] D38589: [Analyzer] Run tests until the end, do not stop at the first failure

[George Karpenkov via Phabricator via cfe-commits]

This revision was automatically updated to reflect the committed changes.
Closed by commit rL314992: [Analyzer Tests] Run static analyzer integration tests until the end, (authored by george.karpenkov).

Changed prior to commit:
  https://reviews.llvm.org/D38589?vs=117847&id=117848#toc

Repository:
  rL LLVM

https://reviews.llvm.org/D38589

Files:
  cfe/trunk/utils/analyzer/SATestBuild.py


Index: cfe/trunk/utils/analyzer/SATestBuild.py
===================================================================
--- cfe/trunk/utils/analyzer/SATestBuild.py
+++ cfe/trunk/utils/analyzer/SATestBuild.py
@@ -570,7 +570,11 @@
       0 - success if there are no crashes or analyzer failure.
       1 - success if there are no difference in the number of reported bugs.
       2 - success if all the bug reports are identical.
+
+    :return success: Whether tests pass according to the Strictness
+    criteria.
     """
+    TestsPassed = True
     TBegin = time.time()
 
     RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
@@ -586,8 +590,6 @@
         RefList.remove(RefLogDir)
     NewList.remove(os.path.join(NewDir, LogFolderName))
 
-    if len(RefList) == 0 or len(NewList) == 0:
-        return False
     assert(len(RefList) == len(NewList))
 
     # There might be more then one folder underneath - one per each scan-build
@@ -624,15 +626,15 @@
                   (NumDiffs, DiffsPath,)
         if Strictness >= 2 and NumDiffs > 0:
             print "Error: Diffs found in strict mode (2)."
-            sys.exit(-1)
+            TestsPassed = False
         elif Strictness >= 1 and ReportsInRef != ReportsInNew:
             print "Error: The number of results are different in "\
                   "strict mode (1)."
-            sys.exit(-1)
+            TestsPassed = False
 
     print "Diagnostic comparison complete (time: %.2f)." % (
           time.time() - TBegin)
-    return (NumDiffs > 0)
+    return TestsPassed
 
 
 def cleanupReferenceResults(SBOutputDir):
@@ -686,6 +688,11 @@
 
 
 def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Strictness=0):
+    """
+    Test a given project.
+    :return TestsPassed: Whether tests have passed according
+    to the :param Strictness: criteria.
+    """
     print " \n\n--- Building project %s" % (ID,)
 
     TBegin = time.time()
@@ -704,11 +711,13 @@
 
     if IsReferenceBuild:
         cleanupReferenceResults(SBOutputDir)
+        TestsPassed = True
     else:
-        runCmpResults(Dir, Strictness)
+        TestsPassed = runCmpResults(Dir, Strictness)
 
     print "Completed tests for project %s (time: %.2f)." % \
           (ID, (time.time() - TBegin))
+    return TestsPassed
 
 
 def isCommentCSVLine(Entries):
@@ -749,6 +758,7 @@
 
 
 def testAll(IsReferenceBuild=False, UpdateSVN=False, Strictness=0):
+    TestsPassed = True
     with projectFileHandler() as PMapFile:
         validateProjectFile(PMapFile)
 
@@ -760,7 +770,7 @@
 
         # Test the projects.
         for (ProjName, ProjBuildMode) in iterateOverProjects(PMapFile):
-            testProject(
+            TestsPassed &= testProject(
                 ProjName, int(ProjBuildMode), IsReferenceBuild, Strictness)
 
         # Re-add reference results to SVN.
@@ -793,4 +803,6 @@
         IsReference = True
         UpdateSVN = True
 
-    testAll(IsReference, UpdateSVN, Strictness)
+    TestsPassed = testAll(IsReference, UpdateSVN, Strictness)
+    if not TestsPassed:
+        sys.exit(-1)


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D38589.117848.patch
Type: text/x-patch
Size: 3084 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20171005/4476d7d7/attachment-0001.bin>